1use crate::account_policy::{redact_identity, AccountPolicyError, AccountPolicyRegistry};
2use crate::compression::CompressedPayload;
3use crate::websocket::admission::{WebSocketAdmissionProvider, WebSocketConnectionPermit};
4use crate::websocket::auth::{AuthContext, AuthDeny, AuthErrorCode};
5use crate::websocket::rate_limiter::{RateLimitResult, WebSocketRateLimiter};
6use arete_auth::Limits;
7use bytes::Bytes;
8use dashmap::DashMap;
9use futures_util::stream::SplitSink;
10use futures_util::SinkExt;
11use std::collections::{HashMap, HashSet};
12use std::net::SocketAddr;
13use std::sync::Arc;
14use std::time::{Duration, Instant, SystemTime};
15use tokio::net::TcpStream;
16use tokio::sync::{mpsc, Mutex as AsyncMutex, RwLock};
17use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode;
18use tokio_tungstenite::tungstenite::protocol::CloseFrame;
19use tokio_tungstenite::{tungstenite::Message, WebSocketStream};
20use tokio_util::sync::CancellationToken;
21use tracing::{debug, info, warn};
22use uuid::Uuid;
23
24pub type WebSocketSender = SplitSink<WebSocketStream<TcpStream>, Message>;
25
26#[derive(Debug, Clone, Copy, PartialEq, Eq)]
28pub enum SendError {
29 ClientNotFound,
31 ClientBackpressured,
33 ClientDisconnected,
35}
36
37impl std::fmt::Display for SendError {
38 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
39 match self {
40 SendError::ClientNotFound => write!(f, "client not found"),
41 SendError::ClientBackpressured => write!(f, "client backpressured and disconnected"),
42 SendError::ClientDisconnected => write!(f, "client disconnected"),
43 }
44 }
45}
46
47impl std::error::Error for SendError {}
48
49#[derive(Debug)]
51struct EgressTracker {
52 bytes_this_minute: u64,
54 window_start: SystemTime,
56}
57
58#[derive(Debug)]
60struct MessageRateTracker {
61 messages_this_minute: u32,
62 window_start: SystemTime,
63}
64
65impl MessageRateTracker {
66 fn new() -> Self {
67 Self {
68 messages_this_minute: 0,
69 window_start: SystemTime::now(),
70 }
71 }
72
73 fn maybe_reset_window(&mut self) {
74 let now = SystemTime::now();
75 if now.duration_since(self.window_start).unwrap_or_default() >= Duration::from_secs(60) {
76 self.messages_this_minute = 0;
77 self.window_start = now;
78 }
79 }
80
81 fn record_message(&mut self, limit: u32) -> bool {
82 self.maybe_reset_window();
83 if self.messages_this_minute + 1 > limit {
84 false
85 } else {
86 self.messages_this_minute += 1;
87 true
88 }
89 }
90
91 fn current_usage(&mut self) -> u32 {
92 self.maybe_reset_window();
93 self.messages_this_minute
94 }
95}
96
97impl EgressTracker {
98 fn new() -> Self {
99 Self {
100 bytes_this_minute: 0,
101 window_start: SystemTime::now(),
102 }
103 }
104
105 fn maybe_reset_window(&mut self) {
107 let now = SystemTime::now();
108 if now.duration_since(self.window_start).unwrap_or_default() >= Duration::from_secs(60) {
109 self.bytes_this_minute = 0;
110 self.window_start = now;
111 }
112 }
113
114 fn record_bytes(&mut self, bytes: usize, limit: u64) -> bool {
116 self.maybe_reset_window();
117 let bytes_u64 = bytes as u64;
118 if self.bytes_this_minute + bytes_u64 > limit {
119 false
120 } else {
121 self.bytes_this_minute += bytes_u64;
122 true
123 }
124 }
125
126 fn current_usage(&mut self) -> u64 {
128 self.maybe_reset_window();
129 self.bytes_this_minute
130 }
131}
132
133#[derive(Debug)]
138struct AccountUsage {
139 egress: std::sync::Mutex<EgressTracker>,
140 messages: std::sync::Mutex<MessageRateTracker>,
141 last_seen: std::sync::Mutex<Instant>,
142}
143
144impl AccountUsage {
145 fn new() -> Self {
146 Self {
147 egress: std::sync::Mutex::new(EgressTracker::new()),
148 messages: std::sync::Mutex::new(MessageRateTracker::new()),
149 last_seen: std::sync::Mutex::new(Instant::now()),
150 }
151 }
152
153 fn touch(&self) {
154 if let Ok(mut last_seen) = self.last_seen.lock() {
155 *last_seen = Instant::now();
156 }
157 }
158
159 fn record_egress(&self, bytes: usize, limit: u64) -> bool {
160 self.touch();
161 match self.egress.lock() {
162 Ok(mut tracker) => tracker.record_bytes(bytes, limit),
163 Err(_) => true,
164 }
165 }
166
167 fn record_message(&self, limit: u32) -> bool {
168 self.touch();
169 match self.messages.lock() {
170 Ok(mut tracker) => tracker.record_message(limit),
171 Err(_) => true,
172 }
173 }
174
175 fn idle_for(&self) -> Duration {
176 self.last_seen
177 .lock()
178 .map(|last_seen| last_seen.elapsed())
179 .unwrap_or(Duration::MAX)
180 }
181}
182
183#[derive(Debug)]
185pub struct ClientInfo {
186 pub id: Uuid,
187 pub last_seen: SystemTime,
188 pub sender: mpsc::Sender<Message>,
189 subscriptions: Arc<RwLock<HashMap<String, CancellationToken>>>,
190 pub auth_context: Option<AuthContext>,
192 pub remote_addr: SocketAddr,
194 egress_tracker: std::sync::Mutex<EgressTracker>,
196 message_rate_tracker: std::sync::Mutex<MessageRateTracker>,
198 close_frame: Arc<std::sync::OnceLock<CloseFrame>>,
202 admission_permit: Option<Arc<dyn WebSocketConnectionPermit>>,
203 admission_lifecycle: Arc<AsyncMutex<()>>,
206}
207
208impl ClientInfo {
209 pub fn new(
210 id: Uuid,
211 sender: mpsc::Sender<Message>,
212 auth_context: Option<AuthContext>,
213 remote_addr: SocketAddr,
214 ) -> Self {
215 Self {
216 id,
217 last_seen: SystemTime::now(),
218 sender,
219 subscriptions: Arc::new(RwLock::new(HashMap::new())),
220 auth_context,
221 remote_addr,
222 egress_tracker: std::sync::Mutex::new(EgressTracker::new()),
223 message_rate_tracker: std::sync::Mutex::new(MessageRateTracker::new()),
224 close_frame: Arc::new(std::sync::OnceLock::new()),
225 admission_permit: None,
226 admission_lifecycle: Arc::new(AsyncMutex::new(())),
227 }
228 }
229
230 pub fn record_egress(&self, bytes: usize) -> Option<u64> {
232 if let Ok(mut tracker) = self.egress_tracker.lock() {
233 if let Some(ref ctx) = self.auth_context {
234 if let Some(limit) = ctx.limits.max_bytes_per_minute {
235 if tracker.record_bytes(bytes, limit) {
236 return Some(tracker.current_usage());
237 } else {
238 return None; }
240 }
241 }
242 return Some(tracker.current_usage());
244 }
245 None
246 }
247
248 pub fn record_inbound_message(&self) -> Option<u32> {
250 if let Ok(mut tracker) = self.message_rate_tracker.lock() {
251 if let Some(ref ctx) = self.auth_context {
252 if let Some(limit) = ctx.limits.max_messages_per_minute {
253 if tracker.record_message(limit) {
254 return Some(tracker.current_usage());
255 } else {
256 return None;
257 }
258 }
259 }
260
261 return Some(tracker.current_usage());
262 }
263
264 None
265 }
266
267 pub fn update_last_seen(&mut self) {
268 self.last_seen = SystemTime::now();
269 }
270
271 pub fn is_stale(&self, timeout: Duration) -> bool {
272 self.last_seen.elapsed().unwrap_or(Duration::MAX) > timeout
273 }
274
275 pub async fn add_subscription(
276 &self,
277 subscription_id: String,
278 token: CancellationToken,
279 ) -> bool {
280 let _lifecycle = self.admission_lifecycle.lock().await;
281 let mut subs = self.subscriptions.write().await;
282 match subs.entry(subscription_id) {
283 std::collections::hash_map::Entry::Vacant(entry) => {
284 entry.insert(token);
285 true
286 }
287 std::collections::hash_map::Entry::Occupied(_) => false,
288 }
289 }
290
291 pub async fn remove_subscription(&self, subscription_id: &str) -> bool {
292 let _lifecycle = self.admission_lifecycle.lock().await;
293 let mut subs = self.subscriptions.write().await;
294 if let Some(token) = subs.remove(subscription_id) {
295 token.cancel();
296 debug!("Cancelled subscription: {}", subscription_id);
297 true
298 } else {
299 debug!(
300 "Subscription not found for cancellation: {}",
301 subscription_id
302 );
303 false
304 }
305 }
306
307 pub async fn cancel_all_subscriptions(&self) {
308 let _lifecycle = self.admission_lifecycle.lock().await;
309 let subs = self.subscriptions.read().await;
310 for (subscription_id, token) in subs.iter() {
311 token.cancel();
312 debug!("Cancelled subscription on disconnect: {}", subscription_id);
313 }
314 }
315
316 pub async fn subscription_count(&self) -> usize {
317 self.subscriptions.read().await.len()
318 }
319}
320
321#[derive(Debug, Clone)]
326pub struct RateLimitConfig {
327 pub max_connections_per_ip: Option<usize>,
329 pub max_connections_per_metering_key: Option<usize>,
331 pub max_connections_per_origin: Option<usize>,
333 pub client_timeout: Duration,
335 pub message_queue_size: usize,
337 pub max_reconnect_attempts: Option<u32>,
339 pub message_rate_window: Duration,
341 pub egress_rate_window: Duration,
343 pub default_limits: Option<Limits>,
346}
347
348impl Default for RateLimitConfig {
349 fn default() -> Self {
350 Self {
351 max_connections_per_ip: None,
352 max_connections_per_metering_key: None,
353 max_connections_per_origin: None,
354 client_timeout: Duration::from_secs(300),
355 message_queue_size: 512,
356 max_reconnect_attempts: None,
357 message_rate_window: Duration::from_secs(60),
358 egress_rate_window: Duration::from_secs(60),
359 default_limits: None,
360 }
361 }
362}
363
364impl RateLimitConfig {
365 pub fn from_env() -> Self {
380 let mut config = Self::default();
381
382 if let Ok(val) = std::env::var("ARETE_WS_MAX_CONNECTIONS_PER_IP") {
383 if let Ok(max) = val.parse() {
384 config.max_connections_per_ip = Some(max);
385 }
386 }
387
388 if let Ok(val) = std::env::var("ARETE_WS_MAX_CONNECTIONS_PER_METERING_KEY") {
389 if let Ok(max) = val.parse() {
390 config.max_connections_per_metering_key = Some(max);
391 }
392 }
393
394 if let Ok(val) = std::env::var("ARETE_WS_MAX_CONNECTIONS_PER_ORIGIN") {
395 if let Ok(max) = val.parse() {
396 config.max_connections_per_origin = Some(max);
397 }
398 }
399
400 if let Ok(val) = std::env::var("ARETE_WS_CLIENT_TIMEOUT_SECS") {
401 if let Ok(secs) = val.parse() {
402 config.client_timeout = Duration::from_secs(secs);
403 }
404 }
405
406 if let Ok(val) = std::env::var("ARETE_WS_MESSAGE_QUEUE_SIZE") {
407 if let Ok(size) = val.parse() {
408 config.message_queue_size = size;
409 }
410 }
411
412 if let Ok(val) = std::env::var("ARETE_WS_RATE_LIMIT_WINDOW_SECS") {
413 if let Ok(secs) = val.parse() {
414 config.message_rate_window = Duration::from_secs(secs);
415 config.egress_rate_window = Duration::from_secs(secs);
416 }
417 }
418
419 let mut default_limits = Limits::default();
421 let mut has_default_limits = false;
422
423 if let Ok(val) = std::env::var("ARETE_WS_DEFAULT_MAX_CONNECTIONS") {
424 if let Ok(max) = val.parse() {
425 default_limits.max_connections = Some(max);
426 has_default_limits = true;
427 }
428 }
429
430 if let Ok(val) = std::env::var("ARETE_WS_DEFAULT_MAX_SUBSCRIPTIONS") {
431 if let Ok(max) = val.parse() {
432 default_limits.max_subscriptions = Some(max);
433 has_default_limits = true;
434 }
435 }
436
437 if let Ok(val) = std::env::var("ARETE_WS_DEFAULT_MAX_SNAPSHOT_ROWS") {
438 if let Ok(max) = val.parse() {
439 default_limits.max_snapshot_rows = Some(max);
440 has_default_limits = true;
441 }
442 }
443
444 if let Ok(val) = std::env::var("ARETE_WS_DEFAULT_MAX_MESSAGES_PER_MINUTE") {
445 if let Ok(max) = val.parse() {
446 default_limits.max_messages_per_minute = Some(max);
447 has_default_limits = true;
448 }
449 }
450
451 if let Ok(val) = std::env::var("ARETE_WS_DEFAULT_MAX_BYTES_PER_MINUTE") {
452 if let Ok(max) = val.parse() {
453 default_limits.max_bytes_per_minute = Some(max);
454 has_default_limits = true;
455 }
456 }
457
458 if has_default_limits {
459 config.default_limits = Some(default_limits);
460 }
461
462 config
463 }
464
465 pub fn with_max_connections_per_ip(mut self, max: usize) -> Self {
467 self.max_connections_per_ip = Some(max);
468 self
469 }
470
471 pub fn with_timeout(mut self, timeout: Duration) -> Self {
473 self.client_timeout = timeout;
474 self
475 }
476
477 pub fn with_message_queue_size(mut self, size: usize) -> Self {
479 self.message_queue_size = size;
480 self
481 }
482
483 pub fn with_rate_limit_window(mut self, window: Duration) -> Self {
485 self.message_rate_window = window;
486 self.egress_rate_window = window;
487 self
488 }
489
490 pub fn with_default_limits(mut self, limits: Limits) -> Self {
495 self.default_limits = Some(limits);
496 self
497 }
498}
499
500const MAX_TRACKED_ACCOUNT_USAGE: usize = crate::account_policy::DEFAULT_MAX_TRACKED_ACCOUNTS;
503
504const ACCOUNT_USAGE_IDLE_TTL: Duration = crate::account_policy::DEFAULT_ACCOUNT_IDLE_TTL;
506
507#[derive(Clone)]
516pub struct ClientManager {
517 clients: Arc<DashMap<Uuid, ClientInfo>>,
518 rate_limit_config: RateLimitConfig,
519 rate_limiter: Option<Arc<WebSocketRateLimiter>>,
521 account_policies: Arc<AccountPolicyRegistry>,
523 account_usage: Arc<DashMap<String, AccountUsage>>,
525 admission_provider: Option<Arc<dyn WebSocketAdmissionProvider>>,
527}
528
529impl ClientManager {
530 pub fn new() -> Self {
531 Self::with_config(RateLimitConfig::default())
532 }
533
534 pub fn with_config(config: RateLimitConfig) -> Self {
536 Self {
537 clients: Arc::new(DashMap::new()),
538 rate_limit_config: config,
539 rate_limiter: None,
540 account_policies: Arc::new(AccountPolicyRegistry::default()),
541 account_usage: Arc::new(DashMap::new()),
542 admission_provider: None,
543 }
544 }
545
546 pub fn with_admission_provider(
548 mut self,
549 provider: Arc<dyn WebSocketAdmissionProvider>,
550 ) -> Self {
551 self.admission_provider = Some(provider);
552 self
553 }
554
555 #[allow(clippy::result_large_err)]
556 pub(crate) fn reserve_connection_admission(
557 &self,
558 context: &AuthContext,
559 ) -> Result<Option<Arc<dyn WebSocketConnectionPermit>>, AuthDeny> {
560 self.admission_provider
561 .as_ref()
562 .map(|provider| provider.reserve_connection(context))
563 .unwrap_or(Ok(None))
564 }
565
566 pub fn from_env() -> Self {
570 Self::with_config(RateLimitConfig::from_env())
571 }
572
573 pub fn with_timeout(mut self, timeout: Duration) -> Self {
575 self.rate_limit_config.client_timeout = timeout;
576 self
577 }
578
579 pub fn with_message_queue_size(mut self, queue_size: usize) -> Self {
581 self.rate_limit_config.message_queue_size = queue_size;
582 self
583 }
584
585 pub fn with_max_connections_per_ip(mut self, max: usize) -> Self {
587 self.rate_limit_config.max_connections_per_ip = Some(max);
588 self
589 }
590
591 pub fn with_rate_limit_window(mut self, window: Duration) -> Self {
593 self.rate_limit_config.message_rate_window = window;
594 self.rate_limit_config.egress_rate_window = window;
595 self
596 }
597
598 pub fn with_default_limits(mut self, limits: Limits) -> Self {
603 self.rate_limit_config.default_limits = Some(limits);
604 self
605 }
606
607 pub fn with_rate_limiter(mut self, rate_limiter: Arc<WebSocketRateLimiter>) -> Self {
609 self.rate_limiter = Some(rate_limiter);
610 self
611 }
612
613 pub fn rate_limiter(&self) -> Option<&WebSocketRateLimiter> {
615 self.rate_limiter.as_ref().map(|r| r.as_ref())
616 }
617
618 pub fn rate_limit_config(&self) -> &RateLimitConfig {
620 &self.rate_limit_config
621 }
622
623 pub fn account_policies(&self) -> &AccountPolicyRegistry {
625 &self.account_policies
626 }
627
628 fn record_account_egress(&self, ctx: &AuthContext, bytes: usize) -> bool {
630 if ctx.is_legacy_policy() || ctx.account_key.is_none() {
631 return true;
632 }
633 let Some(limit) = ctx.account_limits.max_bytes_per_minute else {
634 return true;
635 };
636 let account = ctx.account_key();
637 if let Some(usage) = self.account_usage.get(account) {
638 return usage.record_egress(bytes, limit);
639 }
640 if self.account_usage.len() >= MAX_TRACKED_ACCOUNT_USAGE {
641 warn!(
642 account = %redact_identity(account),
643 "account usage tracking at capacity; aggregate egress guard skipped"
644 );
645 return true;
646 }
647 self.account_usage
648 .entry(account.to_string())
649 .or_insert_with(AccountUsage::new)
650 .record_egress(bytes, limit)
651 }
652
653 fn record_account_message(&self, ctx: &AuthContext) -> bool {
655 if ctx.is_legacy_policy() || ctx.account_key.is_none() {
656 return true;
657 }
658 let Some(limit) = ctx.account_limits.max_messages_per_minute else {
659 return true;
660 };
661 let account = ctx.account_key();
662 if let Some(usage) = self.account_usage.get(account) {
663 return usage.record_message(limit);
664 }
665 if self.account_usage.len() >= MAX_TRACKED_ACCOUNT_USAGE {
666 warn!(
667 account = %redact_identity(account),
668 "account usage tracking at capacity; aggregate message guard skipped"
669 );
670 return true;
671 }
672 self.account_usage
673 .entry(account.to_string())
674 .or_insert_with(AccountUsage::new)
675 .record_message(limit)
676 }
677
678 fn enforce_egress_budgets(&self, client_id: Uuid, bytes: usize) -> Result<(), SendError> {
680 let (connection_ok, account_ok, permit_id) = {
681 let Some(client) = self.clients.get(&client_id) else {
682 return Err(SendError::ClientNotFound);
683 };
684 let connection_ok = client.record_egress(bytes).is_some();
685 let account_ok = !connection_ok
686 || client
687 .auth_context
688 .as_ref()
689 .map(|ctx| self.record_account_egress(ctx, bytes))
690 .unwrap_or(true);
691 let permit_id = client.admission_permit.as_ref().map(|permit| permit.id());
692 (connection_ok, account_ok, permit_id)
693 };
694 if !connection_ok {
695 warn!("Client {} exceeded egress limit, disconnecting", client_id);
696 self.clients.remove(&client_id);
697 return Err(SendError::ClientDisconnected);
698 }
699 if !account_ok {
700 warn!(
701 "Client {} exceeded the account egress limit, disconnecting",
702 client_id
703 );
704 self.clients.remove(&client_id);
705 return Err(SendError::ClientDisconnected);
706 }
707 if let (Some(provider), Some(permit_id)) = (&self.admission_provider, permit_id) {
708 if let Err(deny) = provider.check_egress(permit_id, bytes) {
709 warn!(client = %client_id, reason = %deny.reason, "admission provider denied egress");
710 self.clients.remove(&client_id);
711 return Err(SendError::ClientDisconnected);
712 }
713 }
714 Ok(())
715 }
716
717 #[allow(clippy::result_large_err)]
723 pub fn add_client(
724 &self,
725 client_id: Uuid,
726 ws_sender: WebSocketSender,
727 auth_context: Option<AuthContext>,
728 remote_addr: SocketAddr,
729 ) -> Result<(), AuthDeny> {
730 let permit = if let Some(context) = auth_context.as_ref() {
731 self.reserve_connection_admission(context)?
732 } else {
733 None
734 };
735 self.add_client_with_admission(client_id, ws_sender, auth_context, remote_addr, permit);
736 Ok(())
737 }
738
739 pub(crate) fn add_client_with_admission(
740 &self,
741 client_id: Uuid,
742 mut ws_sender: WebSocketSender,
743 auth_context: Option<AuthContext>,
744 remote_addr: SocketAddr,
745 admission_permit: Option<Arc<dyn WebSocketConnectionPermit>>,
746 ) {
747 let (client_tx, mut client_rx) =
748 mpsc::channel::<Message>(self.rate_limit_config.message_queue_size);
749 let mut client_info = ClientInfo::new(client_id, client_tx, auth_context, remote_addr);
750 client_info.admission_permit = admission_permit;
751 let close_frame = client_info.close_frame.clone();
752
753 let clients_ref = self.clients.clone();
754 tokio::spawn(async move {
755 while let Some(message) = client_rx.recv().await {
756 if let Err(e) = ws_sender.send(message).await {
757 warn!("Failed to send message to client {}: {}", client_id, e);
758 break;
759 }
760 }
761 if let Some(frame) = close_frame.get() {
766 let _ = ws_sender.send(Message::Close(Some(frame.clone()))).await;
767 }
768 let _ = ws_sender.close().await;
769 clients_ref.remove(&client_id);
770 debug!("WebSocket sender task for client {} stopped", client_id);
771 });
772
773 self.clients.insert(client_id, client_info);
774 info!("Client {} registered from {}", client_id, remote_addr);
775 }
776
777 pub fn remove_client(&self, client_id: Uuid) {
779 if self.clients.remove(&client_id).is_some() {
780 info!("Client {} removed", client_id);
781 }
782 }
783
784 pub fn update_client_auth(&self, client_id: Uuid, auth_context: AuthContext) -> bool {
788 self.try_update_client_auth(client_id, auth_context)
789 .unwrap_or(false)
790 }
791
792 #[allow(clippy::result_large_err)]
801 pub fn try_update_client_auth(
802 &self,
803 client_id: Uuid,
804 auth_context: AuthContext,
805 ) -> Result<bool, AuthDeny> {
806 let Some(lifecycle) = self
807 .clients
808 .get(&client_id)
809 .map(|client| client.admission_lifecycle.clone())
810 else {
811 return Ok(false);
812 };
813 let _lifecycle = lifecycle.try_lock().map_err(|_| {
814 AuthDeny::new(
815 AuthErrorCode::InternalError,
816 "Subscription admission is busy",
817 )
818 })?;
819
820 let (subscriptions, current_permit) = {
821 let Some(client) = self.clients.get(&client_id) else {
822 return Ok(false);
823 };
824 if !Arc::ptr_eq(&client.admission_lifecycle, &lifecycle) {
825 return Ok(false);
826 }
827 (
828 client.subscriptions.clone(),
829 client.admission_permit.clone(),
830 )
831 };
832 let subscriptions = subscriptions.try_read().map_err(|_| {
833 AuthDeny::new(
834 AuthErrorCode::InternalError,
835 "Subscription admission is busy",
836 )
837 })?;
838 let active_subscriptions = subscriptions.keys().cloned().collect::<Vec<_>>();
839 drop(subscriptions);
840
841 let permit = match &self.admission_provider {
842 Some(provider) => {
843 provider.refresh_connection(current_permit, &auth_context, &active_subscriptions)?
844 }
845 None => current_permit,
846 };
847
848 let Some(mut client) = self.clients.get_mut(&client_id) else {
849 return Ok(false);
850 };
851 if !Arc::ptr_eq(&client.admission_lifecycle, &lifecycle) {
852 return Ok(false);
853 }
854 client.admission_permit = permit;
855 client.auth_context = Some(auth_context);
856 debug!("Updated auth context for client {}", client_id);
857 Ok(true)
858 }
859
860 pub async fn update_client_auth_async(
864 &self,
865 client_id: Uuid,
866 auth_context: AuthContext,
867 ) -> bool {
868 self.try_update_client_auth_async(client_id, auth_context)
869 .await
870 .unwrap_or(false)
871 }
872
873 #[allow(clippy::result_large_err)]
874 pub(crate) async fn try_update_client_auth_async(
875 &self,
876 client_id: Uuid,
877 auth_context: AuthContext,
878 ) -> Result<bool, AuthDeny> {
879 let Some(lifecycle) = self
880 .clients
881 .get(&client_id)
882 .map(|client| client.admission_lifecycle.clone())
883 else {
884 return Ok(false);
885 };
886 let _lifecycle = lifecycle.lock().await;
887
888 let (subscriptions, current_permit) = {
889 let Some(client) = self.clients.get(&client_id) else {
890 return Ok(false);
891 };
892 if !Arc::ptr_eq(&client.admission_lifecycle, &lifecycle) {
893 return Ok(false);
894 }
895 (
896 client.subscriptions.clone(),
897 client.admission_permit.clone(),
898 )
899 };
900 let subscriptions = subscriptions.read().await;
901 let active_subscriptions = subscriptions.keys().cloned().collect::<Vec<_>>();
902 drop(subscriptions);
903
904 let permit = match &self.admission_provider {
905 Some(provider) => {
906 provider.refresh_connection(current_permit, &auth_context, &active_subscriptions)?
907 }
908 None => current_permit,
909 };
910
911 let Some(mut client) = self.clients.get_mut(&client_id) else {
912 return Ok(false);
913 };
914 if !Arc::ptr_eq(&client.admission_lifecycle, &lifecycle) {
915 return Ok(false);
916 }
917 client.admission_permit = permit;
918 client.auth_context = Some(auth_context);
919 debug!("Updated auth context for client {}", client_id);
920 Ok(true)
921 }
922
923 pub fn check_and_remove_expired(&self, client_id: Uuid) -> bool {
928 let now = std::time::SystemTime::now()
929 .duration_since(std::time::UNIX_EPOCH)
930 .unwrap_or_default()
931 .as_secs();
932 let Some((_, client)) = self.clients.remove_if(&client_id, |_, client| {
936 client
937 .auth_context
938 .as_ref()
939 .is_some_and(|ctx| ctx.expires_at <= now)
940 }) else {
941 return false;
942 };
943 if let Some(ctx) = &client.auth_context {
944 warn!(
945 "Client {} token expired (expired at {}), disconnecting",
946 client_id, ctx.expires_at
947 );
948 }
949 let _ = client.close_frame.set(CloseFrame {
953 code: CloseCode::Policy,
954 reason: format!(
955 "{}: Authentication token expired",
956 AuthErrorCode::TokenExpired.as_str()
957 )
958 .into(),
959 });
960 true
961 }
962
963 pub fn client_count(&self) -> usize {
968 self.clients.len()
969 }
970
971 pub fn send_to_client(&self, client_id: Uuid, data: Arc<Bytes>) -> Result<(), SendError> {
980 if self.check_and_remove_expired(client_id) {
982 return Err(SendError::ClientDisconnected);
983 }
984
985 self.enforce_egress_budgets(client_id, data.len())?;
987
988 let sender = {
989 let client = self
990 .clients
991 .get(&client_id)
992 .ok_or(SendError::ClientNotFound)?;
993 client.sender.clone()
994 };
995
996 let msg = Message::Binary((*data).clone());
997 match sender.try_send(msg) {
998 Ok(()) => Ok(()),
999 Err(mpsc::error::TrySendError::Full(_)) => {
1000 warn!(
1001 "Client {} backpressured (queue full), disconnecting",
1002 client_id
1003 );
1004 self.clients.remove(&client_id);
1005 Err(SendError::ClientBackpressured)
1006 }
1007 Err(mpsc::error::TrySendError::Closed(_)) => {
1008 debug!("Client {} channel closed", client_id);
1009 self.clients.remove(&client_id);
1010 Err(SendError::ClientDisconnected)
1011 }
1012 }
1013 }
1014
1015 pub async fn send_to_client_async(
1024 &self,
1025 client_id: Uuid,
1026 data: Arc<Bytes>,
1027 ) -> Result<(), SendError> {
1028 if self.check_and_remove_expired(client_id) {
1030 return Err(SendError::ClientDisconnected);
1031 }
1032
1033 self.enforce_egress_budgets(client_id, data.len())?;
1035
1036 let sender = {
1037 let client = self
1038 .clients
1039 .get(&client_id)
1040 .ok_or(SendError::ClientNotFound)?;
1041 client.sender.clone()
1042 };
1043
1044 let msg = Message::Binary((*data).clone());
1045 sender
1046 .send(msg)
1047 .await
1048 .map_err(|_| SendError::ClientDisconnected)
1049 }
1050
1051 pub async fn send_text_to_client(
1056 &self,
1057 client_id: Uuid,
1058 text: String,
1059 ) -> Result<(), SendError> {
1060 if self.check_and_remove_expired(client_id) {
1062 return Err(SendError::ClientDisconnected);
1063 }
1064
1065 let sender = {
1066 let client = self
1067 .clients
1068 .get(&client_id)
1069 .ok_or(SendError::ClientNotFound)?;
1070 client.sender.clone()
1071 };
1072
1073 let msg = Message::Text(text.into());
1074 sender
1075 .send(msg)
1076 .await
1077 .map_err(|_| SendError::ClientDisconnected)
1078 }
1079
1080 pub async fn send_compressed_async(
1085 &self,
1086 client_id: Uuid,
1087 payload: CompressedPayload,
1088 ) -> Result<(), SendError> {
1089 if self.check_and_remove_expired(client_id) {
1091 return Err(SendError::ClientDisconnected);
1092 }
1093
1094 let (sender, bytes_to_record) = {
1095 let client = self
1096 .clients
1097 .get(&client_id)
1098 .ok_or(SendError::ClientNotFound)?;
1099
1100 let bytes = match &payload {
1101 CompressedPayload::Compressed(bytes) => bytes.len(),
1102 CompressedPayload::Uncompressed(bytes) => bytes.len(),
1103 };
1104
1105 (client.sender.clone(), bytes)
1106 };
1107
1108 self.enforce_egress_budgets(client_id, bytes_to_record)?;
1110
1111 let msg = match payload {
1112 CompressedPayload::Compressed(bytes) => Message::Binary(bytes),
1113 CompressedPayload::Uncompressed(bytes) => Message::Binary(bytes),
1114 };
1115 sender
1116 .send(msg)
1117 .await
1118 .map_err(|_| SendError::ClientDisconnected)
1119 }
1120
1121 pub fn update_client_last_seen(&self, client_id: Uuid) {
1123 if let Some(mut client) = self.clients.get_mut(&client_id) {
1124 client.update_last_seen();
1125 }
1126 }
1127
1128 #[allow(clippy::result_large_err)]
1130 pub fn check_inbound_message_allowed(&self, client_id: Uuid) -> Result<(), AuthDeny> {
1131 if self.check_and_remove_expired(client_id) {
1132 return Err(AuthDeny::new(
1133 crate::websocket::auth::AuthErrorCode::TokenExpired,
1134 "Authentication token expired",
1135 ));
1136 }
1137
1138 let (connection_ok, account_ok, permit_id) = {
1139 let Some(client) = self.clients.get(&client_id) else {
1140 return Err(AuthDeny::new(
1141 crate::websocket::auth::AuthErrorCode::InternalError,
1142 "Client not found",
1143 ));
1144 };
1145 let connection_ok = client.record_inbound_message().is_some();
1146 let account_ok = !connection_ok
1147 || client
1148 .auth_context
1149 .as_ref()
1150 .map(|ctx| self.record_account_message(ctx))
1151 .unwrap_or(true);
1152 let permit_id = client.admission_permit.as_ref().map(|permit| permit.id());
1153 (connection_ok, account_ok, permit_id)
1154 };
1155
1156 if connection_ok && account_ok {
1157 if let (Some(provider), Some(permit_id)) = (&self.admission_provider, permit_id) {
1158 provider.check_inbound_message(permit_id)?;
1159 }
1160 return Ok(());
1161 }
1162 let scope = if connection_ok {
1163 "inbound account websocket messages"
1164 } else {
1165 "inbound websocket messages"
1166 };
1167 Err(
1168 AuthDeny::rate_limited(self.rate_limit_config.message_rate_window, scope).with_context(
1169 format!("client {} exceeded the inbound message budget", client_id),
1170 ),
1171 )
1172 }
1173
1174 pub fn has_client(&self, client_id: Uuid) -> bool {
1176 self.clients.contains_key(&client_id)
1177 }
1178
1179 pub async fn add_client_subscription(
1180 &self,
1181 client_id: Uuid,
1182 subscription_id: String,
1183 token: CancellationToken,
1184 ) -> bool {
1185 self.try_add_client_subscription(client_id, subscription_id, token)
1186 .await
1187 .unwrap_or(false)
1188 }
1189
1190 #[allow(clippy::result_large_err)]
1191 pub(crate) async fn try_add_client_subscription(
1192 &self,
1193 client_id: Uuid,
1194 subscription_id: String,
1195 token: CancellationToken,
1196 ) -> Result<bool, AuthDeny> {
1197 let Some(lifecycle) = self
1198 .clients
1199 .get(&client_id)
1200 .map(|client| client.admission_lifecycle.clone())
1201 else {
1202 return Ok(false);
1203 };
1204 let _lifecycle = lifecycle.lock().await;
1205 let Some((subscriptions, permit)) = self.clients.get(&client_id).and_then(|client| {
1206 Arc::ptr_eq(&client.admission_lifecycle, &lifecycle).then(|| {
1207 (
1208 client.subscriptions.clone(),
1209 client.admission_permit.clone(),
1210 )
1211 })
1212 }) else {
1213 return Ok(false);
1214 };
1215 let permit_id = permit.as_ref().map(|permit| permit.id());
1216 if let (Some(provider), Some(permit_id)) = (&self.admission_provider, permit_id) {
1217 if !provider.add_subscription(permit_id, &subscription_id)? {
1218 return Ok(false);
1219 }
1220 }
1221 let mut subscriptions = subscriptions.write().await;
1222 let client = self.clients.get(&client_id);
1223 let registered = client
1224 .as_ref()
1225 .is_some_and(|client| Arc::ptr_eq(&client.admission_lifecycle, &lifecycle));
1226 if !registered || subscriptions.contains_key(&subscription_id) {
1227 drop(client);
1228 drop(subscriptions);
1229 if let (Some(provider), Some(permit_id)) = (&self.admission_provider, permit_id) {
1230 provider.remove_subscription(permit_id, &subscription_id);
1231 }
1232 return Ok(false);
1233 }
1234 subscriptions.insert(subscription_id, token);
1235 drop(client);
1236 Ok(true)
1237 }
1238
1239 pub async fn remove_client_subscription(&self, client_id: Uuid, subscription_id: &str) -> bool {
1240 let Some(lifecycle) = self
1241 .clients
1242 .get(&client_id)
1243 .map(|client| client.admission_lifecycle.clone())
1244 else {
1245 return false;
1246 };
1247 let _lifecycle = lifecycle.lock().await;
1248 let Some((subscriptions, permit)) = self.clients.get(&client_id).and_then(|client| {
1249 Arc::ptr_eq(&client.admission_lifecycle, &lifecycle).then(|| {
1250 (
1251 client.subscriptions.clone(),
1252 client.admission_permit.clone(),
1253 )
1254 })
1255 }) else {
1256 return false;
1257 };
1258 let permit_id = permit.as_ref().map(|permit| permit.id());
1259 let mut subscriptions = subscriptions.write().await;
1260 let client = self.clients.get(&client_id);
1261 if !client
1262 .as_ref()
1263 .is_some_and(|client| Arc::ptr_eq(&client.admission_lifecycle, &lifecycle))
1264 {
1265 return false;
1266 }
1267 let token = subscriptions.remove(subscription_id);
1268 drop(client);
1269 drop(subscriptions);
1270 if let Some(token) = token {
1271 token.cancel();
1272 if let (Some(provider), Some(permit_id)) = (&self.admission_provider, permit_id) {
1273 provider.remove_subscription(permit_id, subscription_id);
1274 }
1275 true
1276 } else {
1277 false
1278 }
1279 }
1280
1281 pub async fn cancel_all_client_subscriptions(&self, client_id: Uuid) {
1282 let Some(lifecycle) = self
1283 .clients
1284 .get(&client_id)
1285 .map(|client| client.admission_lifecycle.clone())
1286 else {
1287 return;
1288 };
1289 let _lifecycle = lifecycle.lock().await;
1290 let Some(subscriptions) = self.clients.get(&client_id).and_then(|client| {
1291 Arc::ptr_eq(&client.admission_lifecycle, &lifecycle)
1292 .then(|| client.subscriptions.clone())
1293 }) else {
1294 return;
1295 };
1296 let subscriptions = subscriptions.read().await;
1297 for (subscription_id, token) in subscriptions.iter() {
1298 token.cancel();
1299 debug!("Cancelled subscription on disconnect: {}", subscription_id);
1300 }
1301 }
1302
1303 pub fn cleanup_stale_clients(&self) -> usize {
1305 let timeout = self.rate_limit_config.client_timeout;
1306 let mut stale_clients = Vec::new();
1307
1308 for entry in self.clients.iter() {
1309 if entry.value().is_stale(timeout) {
1310 stale_clients.push(*entry.key());
1311 }
1312 }
1313
1314 let removed_count = stale_clients.len();
1315 for client_id in stale_clients {
1316 self.clients.remove(&client_id);
1317 info!("Removed stale client {}", client_id);
1318 }
1319
1320 removed_count
1321 }
1322
1323 fn cleanup_account_state(&self) {
1326 let live_accounts: HashSet<String> = self
1327 .clients
1328 .iter()
1329 .filter_map(|entry| {
1330 entry
1331 .value()
1332 .auth_context
1333 .as_ref()
1334 .and_then(|ctx| ctx.account_key.clone())
1335 })
1336 .collect();
1337 self.account_usage.retain(|account, usage| {
1338 live_accounts.contains(account) || usage.idle_for() < ACCOUNT_USAGE_IDLE_TTL
1339 });
1340 self.account_policies
1341 .evict_idle(|account| live_accounts.contains(account));
1342 }
1343
1344 pub fn start_cleanup_task(&self) -> tokio::task::JoinHandle<()> {
1349 let client_manager = self.clone();
1350
1351 tokio::spawn(async move {
1352 let mut interval = tokio::time::interval(Duration::from_secs(30));
1353
1354 loop {
1355 interval.tick().await;
1356 let removed = client_manager.cleanup_stale_clients();
1357 if removed > 0 {
1358 info!("Cleaned up {} stale clients", removed);
1359 }
1360 client_manager.cleanup_account_state();
1361 if let Some(provider) = &client_manager.admission_provider {
1362 provider.cleanup();
1363 }
1364 }
1365 })
1366 }
1367
1368 #[allow(clippy::result_large_err)]
1376 pub async fn check_connection_allowed(
1377 &self,
1378 remote_addr: SocketAddr,
1379 auth_context: &Option<AuthContext>,
1380 ) -> Result<(), AuthDeny> {
1381 if let Some(ref rate_limiter) = self.rate_limiter {
1383 match rate_limiter.check_handshake(remote_addr).await {
1385 RateLimitResult::Allowed { .. } => {}
1386 RateLimitResult::Denied { retry_after, limit } => {
1387 return Err(AuthDeny::rate_limited(retry_after, "websocket handshakes")
1388 .with_context(format!(
1389 "handshake rate limit of {} per minute exceeded for {}",
1390 limit, remote_addr
1391 )));
1392 }
1393 }
1394
1395 if let Some(ref ctx) = auth_context {
1396 match rate_limiter
1400 .check_connection_for_consumer(
1401 ctx.consumer_key(),
1402 ctx.limits.max_connection_attempts_per_minute,
1403 )
1404 .await
1405 {
1406 RateLimitResult::Allowed { .. } => {}
1407 RateLimitResult::Denied { retry_after, limit } => {
1408 return Err(AuthDeny::rate_limited(retry_after, "websocket connections")
1409 .with_context(format!(
1410 "connection rate limit for consumer {} of {} per minute exceeded",
1411 ctx.consumer_key(),
1412 limit
1413 )));
1414 }
1415 }
1416
1417 if ctx.is_legacy_policy() || ctx.account_key.is_some() {
1421 match rate_limiter
1422 .check_connection_for_account(
1423 ctx.account_key(),
1424 ctx.account_limits.max_connection_attempts_per_minute,
1425 )
1426 .await
1427 {
1428 RateLimitResult::Allowed { .. } => {}
1429 RateLimitResult::Denied { retry_after, limit } => {
1430 return Err(AuthDeny::rate_limited(
1431 retry_after,
1432 "metered websocket connections",
1433 )
1434 .with_context(format!(
1435 "connection rate limit for account {} of {} per minute exceeded",
1436 ctx.account_key(),
1437 limit
1438 )));
1439 }
1440 }
1441 }
1442 }
1443 }
1444
1445 if let Some(max_per_ip) = self.rate_limit_config.max_connections_per_ip {
1447 let current_ip_connections = self.count_connections_for_ip(&remote_addr);
1448 if current_ip_connections >= max_per_ip {
1449 return Err(AuthDeny::connection_limit_exceeded(
1450 &format!("ip {}", remote_addr.ip()),
1451 current_ip_connections,
1452 max_per_ip,
1453 ));
1454 }
1455 }
1456
1457 if let Some(ctx) = auth_context {
1458 if ctx.is_legacy_policy() {
1461 let legacy_policy_token = self.account_policies.record_legacy_token();
1462 debug!(legacy_policy_token, "legacy policy token admitted");
1463 } else if let (Some(account), Some(policy_version)) =
1464 (ctx.account_key.as_deref(), ctx.policy_version)
1465 {
1466 match self
1467 .account_policies
1468 .observe(account, policy_version, &ctx.account_limits)
1469 {
1470 Ok(()) => {}
1471 Err(AccountPolicyError::StaleVersion { presented, current }) => {
1472 debug!(
1473 account = %redact_identity(account),
1474 presented,
1475 current,
1476 "stale policy version rejected"
1477 );
1478 return Err(AuthDeny::new(
1479 crate::websocket::auth::AuthErrorCode::TokenExpired,
1480 "session policy version is stale; refresh the session token",
1481 ));
1482 }
1483 Err(AccountPolicyError::ConflictingLimits { version }) => {
1484 warn!(
1485 account = %redact_identity(account),
1486 version,
1487 "signed account limits conflict for one policy version"
1488 );
1489 return Err(AuthDeny::new(
1490 crate::websocket::auth::AuthErrorCode::InternalError,
1491 "signed account limits conflict with previously observed policy",
1492 ));
1493 }
1494 Err(AccountPolicyError::CapacityExhausted) => {
1495 warn!("account policy state at capacity; denying admission");
1496 return Err(AuthDeny::new(
1497 crate::websocket::auth::AuthErrorCode::InternalError,
1498 "account policy state is at capacity; retry shortly",
1499 ));
1500 }
1501 }
1502 }
1503
1504 let max_connections = ctx.limits.max_connections.or_else(|| {
1508 self.rate_limit_config
1509 .default_limits
1510 .as_ref()
1511 .and_then(|l| l.max_connections)
1512 });
1513 if let Some(max_connections) = max_connections {
1514 let current_connections = self.count_connections_for_consumer(ctx.consumer_key());
1515 if current_connections >= max_connections as usize {
1516 return Err(AuthDeny::connection_limit_exceeded(
1517 &format!("consumer {}", ctx.consumer_key()),
1518 current_connections,
1519 max_connections as usize,
1520 ));
1521 }
1522 }
1523
1524 if !ctx.is_legacy_policy() && ctx.account_key.is_some() {
1527 if let Some(max_account_connections) = ctx.account_limits.max_connections {
1528 let current_account_connections =
1529 self.count_connections_for_account(ctx.account_key());
1530 if current_account_connections >= max_account_connections as usize {
1531 return Err(AuthDeny::connection_limit_exceeded(
1532 &format!("account {}", ctx.account_key()),
1533 current_account_connections,
1534 max_account_connections as usize,
1535 ));
1536 }
1537 }
1538 }
1539
1540 if let Some(max_per_metering_key) =
1542 self.rate_limit_config.max_connections_per_metering_key
1543 {
1544 let current_metering_connections =
1545 self.count_connections_for_metering_key(&ctx.metering_key);
1546 if current_metering_connections >= max_per_metering_key {
1547 return Err(AuthDeny::connection_limit_exceeded(
1548 &format!("metering key {}", ctx.metering_key),
1549 current_metering_connections,
1550 max_per_metering_key,
1551 ));
1552 }
1553 }
1554
1555 if let Some(max_per_origin) = self.rate_limit_config.max_connections_per_origin {
1557 if let Some(ref origin) = ctx.origin {
1558 let current_origin_connections = self.count_connections_for_origin(origin);
1559 if current_origin_connections >= max_per_origin {
1560 return Err(AuthDeny::connection_limit_exceeded(
1561 &format!("origin {}", origin),
1562 current_origin_connections,
1563 max_per_origin,
1564 ));
1565 }
1566 }
1567 }
1568 }
1569 Ok(())
1570 }
1571
1572 fn count_connections_for_ip(&self, remote_addr: &SocketAddr) -> usize {
1574 let ip = remote_addr.ip();
1575 self.clients
1576 .iter()
1577 .filter(|entry| entry.value().remote_addr.ip() == ip)
1578 .count()
1579 }
1580
1581 fn count_connections_for_consumer(&self, consumer: &str) -> usize {
1583 self.clients
1584 .iter()
1585 .filter(|entry| {
1586 entry
1587 .value()
1588 .auth_context
1589 .as_ref()
1590 .map(|ctx| ctx.consumer_key() == consumer)
1591 .unwrap_or(false)
1592 })
1593 .count()
1594 }
1595
1596 fn count_connections_for_account(&self, account: &str) -> usize {
1598 self.clients
1599 .iter()
1600 .filter(|entry| {
1601 entry
1602 .value()
1603 .auth_context
1604 .as_ref()
1605 .map(|ctx| ctx.account_key() == account)
1606 .unwrap_or(false)
1607 })
1608 .count()
1609 }
1610
1611 fn count_connections_for_metering_key(&self, metering_key: &str) -> usize {
1613 self.clients
1614 .iter()
1615 .filter(|entry| {
1616 entry
1617 .value()
1618 .auth_context
1619 .as_ref()
1620 .map(|ctx| ctx.metering_key == metering_key)
1621 .unwrap_or(false)
1622 })
1623 .count()
1624 }
1625
1626 fn count_connections_for_origin(&self, origin: &str) -> usize {
1628 self.clients
1629 .iter()
1630 .filter(|entry| {
1631 entry
1632 .value()
1633 .auth_context
1634 .as_ref()
1635 .and_then(|ctx| ctx.origin.as_ref())
1636 .map(|o| o == origin)
1637 .unwrap_or(false)
1638 })
1639 .count()
1640 }
1641
1642 #[allow(clippy::result_large_err)]
1646 pub async fn check_subscription_allowed(&self, client_id: Uuid) -> Result<(), AuthDeny> {
1647 let context = {
1648 let Some(client) = self.clients.get(&client_id) else {
1649 return Ok(());
1650 };
1651 let current_subs = client.subscription_count().await;
1652
1653 if let Some(ref ctx) = client.auth_context {
1655 let max_subs = ctx.limits.max_subscriptions.or_else(|| {
1656 self.rate_limit_config
1657 .default_limits
1658 .as_ref()
1659 .and_then(|l| l.max_subscriptions)
1660 });
1661 if let Some(max_subs) = max_subs {
1662 if current_subs >= max_subs as usize {
1663 return Err(AuthDeny::new(
1664 crate::websocket::auth::AuthErrorCode::SubscriptionLimitExceeded,
1665 format!(
1666 "Subscription limit exceeded: {} of {} subscriptions for client {}",
1667 current_subs, max_subs, client_id
1668 ),
1669 )
1670 .with_suggested_action(
1671 "Unsubscribe from an existing view before creating another subscription",
1672 ));
1673 }
1674 }
1675 }
1676 client.auth_context.clone()
1677 };
1678
1679 if let (Some(rate_limiter), Some(ctx)) = (self.rate_limiter.as_ref(), context.as_ref()) {
1682 if !ctx.is_legacy_policy() {
1683 if let RateLimitResult::Denied { retry_after, limit } = rate_limiter
1684 .check_subscription_create_for_consumer(
1685 ctx.consumer_key(),
1686 ctx.limits.max_subscription_creates_per_minute,
1687 )
1688 .await
1689 {
1690 return Err(AuthDeny::rate_limited(retry_after, "subscription creates")
1691 .with_context(format!(
1692 "subscription-create rate limit for consumer {} of {} per minute exceeded",
1693 ctx.consumer_key(),
1694 limit
1695 )));
1696 }
1697
1698 if ctx.account_key.is_some() {
1699 if let RateLimitResult::Denied { retry_after, limit } = rate_limiter
1700 .check_subscription_create_for_account(
1701 ctx.account_key(),
1702 ctx.account_limits.max_subscription_creates_per_minute,
1703 )
1704 .await
1705 {
1706 return Err(AuthDeny::rate_limited(
1707 retry_after,
1708 "account subscription creates",
1709 )
1710 .with_context(format!(
1711 "subscription-create rate limit for account {} of {} per minute exceeded",
1712 ctx.account_key(),
1713 limit
1714 )));
1715 }
1716 }
1717 }
1718 }
1719 Ok(())
1720 }
1721
1722 pub fn get_metering_key(&self, client_id: Uuid) -> Option<String> {
1724 self.clients.get(&client_id).and_then(|client| {
1725 client
1726 .auth_context
1727 .as_ref()
1728 .map(|ctx| ctx.metering_key.clone())
1729 })
1730 }
1731
1732 pub fn get_auth_context(&self, client_id: Uuid) -> Option<AuthContext> {
1734 self.clients
1735 .get(&client_id)
1736 .and_then(|client| client.auth_context.clone())
1737 }
1738
1739 #[allow(clippy::result_large_err)]
1743 pub fn check_snapshot_allowed(
1744 &self,
1745 client_id: Uuid,
1746 requested_rows: u32,
1747 ) -> Result<(), AuthDeny> {
1748 if let Some(client) = self.clients.get(&client_id) {
1749 if let Some(ref ctx) = client.auth_context {
1750 let max_rows = ctx.limits.max_snapshot_rows.or_else(|| {
1751 self.rate_limit_config
1752 .default_limits
1753 .as_ref()
1754 .and_then(|l| l.max_snapshot_rows)
1755 });
1756 if let Some(max_rows) = max_rows {
1757 if requested_rows > max_rows {
1758 return Err(AuthDeny::new(
1759 crate::websocket::auth::AuthErrorCode::SnapshotLimitExceeded,
1760 format!(
1761 "Snapshot limit exceeded: requested {} rows, max allowed is {} for client {}",
1762 requested_rows, max_rows, client_id
1763 ),
1764 )
1765 .with_suggested_action(
1766 "Request fewer rows or lower the snapshotLimit on the subscription",
1767 ));
1768 }
1769 }
1770 }
1771 }
1772 Ok(())
1773 }
1774}
1775
1776impl Default for ClientManager {
1777 fn default() -> Self {
1778 Self::new()
1779 }
1780}
1781
1782#[cfg(test)]
1783#[path = "admission_tests.rs"]
1784mod admission_tests;
1785
1786#[cfg(test)]
1787mod tests {
1788 use super::*;
1789 use crate::websocket::auth::AuthContext;
1790 use arete_auth::{KeyClass, Limits};
1791 use std::net::{IpAddr, Ipv4Addr, SocketAddr};
1792
1793 fn create_test_auth_context(subject: &str, limits: Limits) -> AuthContext {
1794 AuthContext {
1795 subject: subject.to_string(),
1796 issuer: "test-issuer".to_string(),
1797 audience: "test-audience".to_string(),
1798 key_class: KeyClass::Publishable,
1799 metering_key: format!("meter-{}", subject),
1800 deployment_id: None,
1801 target_kind: None,
1802 target_id: None,
1803 program_id: None,
1804 program_release_hash: None,
1805 expires_at: u64::MAX,
1806 scope: "read".to_string(),
1807 limits,
1808 plan: None,
1809 origin: None,
1810 client_ip: None,
1811 jti: uuid::Uuid::new_v4().to_string(),
1812 actor_key: None,
1813 account_key: None,
1814 consumer_key: None,
1815 policy_version: None,
1816 account_limits: Limits::default(),
1817 }
1818 }
1819
1820 fn create_v2_auth_context(
1821 consumer: &str,
1822 account: &str,
1823 policy_version: u32,
1824 limits: Limits,
1825 account_limits: Limits,
1826 ) -> AuthContext {
1827 AuthContext {
1828 subject: "user:1".to_string(),
1829 issuer: "test-issuer".to_string(),
1830 audience: "test-audience".to_string(),
1831 key_class: KeyClass::Publishable,
1832 metering_key: account.to_string(),
1833 deployment_id: None,
1834 target_kind: None,
1835 target_id: None,
1836 program_id: None,
1837 program_release_hash: None,
1838 expires_at: u64::MAX,
1839 scope: "read".to_string(),
1840 limits,
1841 plan: Some("pro".to_string()),
1842 origin: None,
1843 client_ip: None,
1844 jti: uuid::Uuid::new_v4().to_string(),
1845 actor_key: Some("user:1".to_string()),
1846 account_key: Some(account.to_string()),
1847 consumer_key: Some(consumer.to_string()),
1848 policy_version: Some(policy_version),
1849 account_limits,
1850 }
1851 }
1852
1853 fn insert_client(manager: &ClientManager, context: AuthContext) -> Uuid {
1854 let (sender, receiver) = mpsc::channel(8);
1855 std::mem::forget(receiver);
1857 let client_id = Uuid::new_v4();
1858 manager.clients.insert(
1859 client_id,
1860 ClientInfo::new(
1861 client_id,
1862 sender,
1863 Some(context),
1864 create_test_socket_addr("127.0.0.1"),
1865 ),
1866 );
1867 client_id
1868 }
1869
1870 fn create_test_socket_addr(ip: &str) -> SocketAddr {
1871 SocketAddr::new(
1872 ip.parse::<IpAddr>()
1873 .unwrap_or(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))),
1874 12345,
1875 )
1876 }
1877
1878 #[test]
1879 fn test_egress_tracker_basic() {
1880 let mut tracker = EgressTracker::new();
1881
1882 assert!(tracker.record_bytes(500, 1000));
1884 assert_eq!(tracker.current_usage(), 500);
1885
1886 assert!(tracker.record_bytes(400, 1000));
1888 assert_eq!(tracker.current_usage(), 900);
1889
1890 assert!(!tracker.record_bytes(200, 1000));
1892 assert_eq!(tracker.current_usage(), 900); }
1894
1895 #[test]
1896 fn test_egress_tracker_window_reset() {
1897 let mut tracker = EgressTracker::new();
1898
1899 assert!(tracker.record_bytes(100, 100));
1901 assert!(!tracker.record_bytes(1, 100));
1902
1903 tracker.bytes_this_minute = 0;
1905 tracker.window_start = SystemTime::now() - Duration::from_secs(61);
1906
1907 assert!(tracker.record_bytes(50, 100));
1909 }
1910
1911 #[test]
1912 fn test_message_rate_tracker_basic() {
1913 let mut tracker = MessageRateTracker::new();
1914
1915 assert!(tracker.record_message(2));
1916 assert_eq!(tracker.current_usage(), 1);
1917
1918 assert!(tracker.record_message(2));
1919 assert_eq!(tracker.current_usage(), 2);
1920
1921 assert!(!tracker.record_message(2));
1922 assert_eq!(tracker.current_usage(), 2);
1923 }
1924
1925 #[tokio::test]
1926 async fn test_client_inbound_message_limit() {
1927 let (tx, _rx) = mpsc::channel(1);
1928 let client = ClientInfo::new(
1929 Uuid::new_v4(),
1930 tx,
1931 Some(create_test_auth_context(
1932 "user-1",
1933 Limits {
1934 max_messages_per_minute: Some(2),
1935 ..Default::default()
1936 },
1937 )),
1938 create_test_socket_addr("127.0.0.1"),
1939 );
1940
1941 assert_eq!(client.record_inbound_message(), Some(1));
1942 assert_eq!(client.record_inbound_message(), Some(2));
1943 assert_eq!(client.record_inbound_message(), None);
1944 }
1945
1946 #[tokio::test]
1947 async fn duplicate_subscription_id_is_rejected_without_replacement() {
1948 let (tx, _rx) = mpsc::channel(1);
1949 let client = ClientInfo::new(
1950 Uuid::new_v4(),
1951 tx,
1952 None,
1953 create_test_socket_addr("127.0.0.1"),
1954 );
1955 let first = CancellationToken::new();
1956 let duplicate = CancellationToken::new();
1957
1958 assert!(
1959 client
1960 .add_subscription("opaque-id".to_string(), first.clone())
1961 .await
1962 );
1963 assert!(
1964 !client
1965 .add_subscription("opaque-id".to_string(), duplicate.clone())
1966 .await
1967 );
1968 assert!(!first.is_cancelled());
1969 assert!(!duplicate.is_cancelled());
1970
1971 assert!(client.remove_subscription("opaque-id").await);
1972 assert!(first.is_cancelled());
1973 assert!(!duplicate.is_cancelled());
1974 }
1975
1976 #[tokio::test]
1977 async fn test_no_limits() {
1978 let manager = ClientManager::new();
1979 let addr = create_test_socket_addr("127.0.0.1");
1980
1981 assert!(manager.check_connection_allowed(addr, &None).await.is_ok());
1983
1984 let auth_context = create_test_auth_context("test", Limits::default());
1986 assert!(manager
1987 .check_connection_allowed(addr, &Some(auth_context))
1988 .await
1989 .is_ok());
1990 }
1991
1992 #[tokio::test]
1993 async fn test_per_subject_connection_limit() {
1994 let manager = ClientManager::new();
1995
1996 let limits = Limits {
1997 max_connections: Some(2),
1998 ..Default::default()
1999 };
2000
2001 let auth_context = create_test_auth_context("user-1", limits);
2002 let addr = create_test_socket_addr("127.0.0.1");
2003
2004 assert!(manager
2006 .check_connection_allowed(addr, &Some(auth_context.clone()))
2007 .await
2008 .is_ok());
2009 }
2010
2011 #[tokio::test]
2012 async fn test_per_ip_connection_limit() {
2013 let manager = ClientManager::new().with_max_connections_per_ip(2);
2014 let addr = create_test_socket_addr("192.168.1.1");
2015
2016 assert!(manager.check_connection_allowed(addr, &None).await.is_ok());
2018 }
2019
2020 #[test]
2022 fn rate_limit_config_default() {
2023 let config = RateLimitConfig::default();
2024 assert!(config.max_connections_per_ip.is_none());
2025 assert_eq!(config.client_timeout, Duration::from_secs(300));
2026 assert_eq!(config.message_queue_size, 512);
2027 assert!(config.max_reconnect_attempts.is_none());
2028 assert_eq!(config.message_rate_window, Duration::from_secs(60));
2029 assert_eq!(config.egress_rate_window, Duration::from_secs(60));
2030 }
2031
2032 #[test]
2033 fn rate_limit_config_builder_methods() {
2034 let config = RateLimitConfig::default()
2035 .with_max_connections_per_ip(10)
2036 .with_timeout(Duration::from_secs(600))
2037 .with_message_queue_size(1024)
2038 .with_rate_limit_window(Duration::from_secs(120));
2039
2040 assert_eq!(config.max_connections_per_ip, Some(10));
2041 assert_eq!(config.client_timeout, Duration::from_secs(600));
2042 assert_eq!(config.message_queue_size, 1024);
2043 assert_eq!(config.message_rate_window, Duration::from_secs(120));
2044 assert_eq!(config.egress_rate_window, Duration::from_secs(120));
2045 }
2046
2047 #[tokio::test]
2048 async fn client_manager_with_config() {
2049 let config = RateLimitConfig::default()
2050 .with_max_connections_per_ip(5)
2051 .with_timeout(Duration::from_secs(120))
2052 .with_message_queue_size(256);
2053
2054 let manager = ClientManager::with_config(config);
2055 let addr = create_test_socket_addr("10.0.0.1");
2056
2057 assert_eq!(manager.rate_limit_config().max_connections_per_ip, Some(5));
2059 assert_eq!(
2060 manager.rate_limit_config().client_timeout,
2061 Duration::from_secs(120)
2062 );
2063 assert_eq!(manager.rate_limit_config().message_queue_size, 256);
2064
2065 assert!(manager.check_connection_allowed(addr, &None).await.is_ok());
2067 }
2068
2069 #[tokio::test]
2070 async fn client_manager_builder_pattern() {
2071 let manager = ClientManager::new()
2072 .with_max_connections_per_ip(10)
2073 .with_timeout(Duration::from_secs(180))
2074 .with_message_queue_size(1024)
2075 .with_rate_limit_window(Duration::from_secs(90));
2076
2077 assert_eq!(manager.rate_limit_config().max_connections_per_ip, Some(10));
2078 assert_eq!(
2079 manager.rate_limit_config().client_timeout,
2080 Duration::from_secs(180)
2081 );
2082 assert_eq!(manager.rate_limit_config().message_queue_size, 1024);
2083 assert_eq!(
2084 manager.rate_limit_config().message_rate_window,
2085 Duration::from_secs(90)
2086 );
2087 }
2088
2089 #[tokio::test]
2091 async fn connection_limit_enforcement_with_actual_clients() {
2092 let manager = ClientManager::new().with_max_connections_per_ip(2);
2093 let addr1 = create_test_socket_addr("192.168.1.1");
2094 let addr2 = create_test_socket_addr("192.168.1.2");
2095
2096 let auth1 = create_test_auth_context("user-1", Limits::default());
2098 assert!(manager
2099 .check_connection_allowed(addr1, &Some(auth1.clone()))
2100 .await
2101 .is_ok());
2102
2103 let auth2 = create_test_auth_context("user-2", Limits::default());
2108 assert!(manager
2109 .check_connection_allowed(addr1, &Some(auth2.clone()))
2110 .await
2111 .is_ok());
2112
2113 let auth3 = create_test_auth_context("user-3", Limits::default());
2115 assert!(manager
2116 .check_connection_allowed(addr2, &Some(auth3.clone()))
2117 .await
2118 .is_ok());
2119 }
2120
2121 #[tokio::test]
2123 async fn subscription_limit_enforcement() {
2124 let manager = ClientManager::new();
2125 let addr = create_test_socket_addr("127.0.0.1");
2126
2127 let auth = create_test_auth_context(
2129 "user-1",
2130 Limits {
2131 max_subscriptions: Some(2),
2132 ..Default::default()
2133 },
2134 );
2135
2136 assert!(manager
2138 .check_connection_allowed(addr, &Some(auth.clone()))
2139 .await
2140 .is_ok());
2141
2142 assert_eq!(auth.limits.max_subscriptions, Some(2));
2145 }
2146
2147 #[tokio::test]
2149 async fn snapshot_limit_enforcement() {
2150 let manager = ClientManager::new();
2151 let addr = create_test_socket_addr("127.0.0.1");
2152
2153 let auth = create_test_auth_context(
2154 "user-1",
2155 Limits {
2156 max_snapshot_rows: Some(1000),
2157 ..Default::default()
2158 },
2159 );
2160
2161 assert!(manager
2162 .check_connection_allowed(addr, &Some(auth.clone()))
2163 .await
2164 .is_ok());
2165
2166 }
2169
2170 #[tokio::test]
2171 async fn two_consumers_share_the_account_connection_cap() {
2172 let manager = ClientManager::new();
2173 let account_limits = Limits {
2174 max_connections: Some(1),
2175 ..Limits::default()
2176 };
2177 let consumer_a = create_v2_auth_context(
2178 "consumer:a",
2179 "account:42",
2180 1,
2181 Limits::default(),
2182 account_limits.clone(),
2183 );
2184 insert_client(&manager, consumer_a);
2185
2186 let consumer_b = create_v2_auth_context(
2189 "consumer:b",
2190 "account:42",
2191 1,
2192 Limits::default(),
2193 account_limits.clone(),
2194 );
2195 let addr = create_test_socket_addr("127.0.0.1");
2196 assert!(manager
2197 .check_connection_allowed(addr, &Some(consumer_b))
2198 .await
2199 .is_err());
2200
2201 let other_account = create_v2_auth_context(
2203 "consumer:c",
2204 "account:43",
2205 1,
2206 Limits::default(),
2207 account_limits,
2208 );
2209 assert!(manager
2210 .check_connection_allowed(addr, &Some(other_account))
2211 .await
2212 .is_ok());
2213 }
2214
2215 #[tokio::test]
2216 async fn one_consumer_is_limited_independently_of_its_account() {
2217 let manager = ClientManager::new();
2218 let limits = Limits {
2219 max_connections: Some(1),
2220 ..Limits::default()
2221 };
2222 let account_limits = Limits {
2223 max_connections: Some(10),
2224 ..Limits::default()
2225 };
2226 let consumer_a = create_v2_auth_context(
2227 "consumer:a",
2228 "account:42",
2229 1,
2230 limits.clone(),
2231 account_limits.clone(),
2232 );
2233 insert_client(&manager, consumer_a.clone());
2234
2235 let addr = create_test_socket_addr("127.0.0.1");
2236 assert!(manager
2238 .check_connection_allowed(addr, &Some(consumer_a))
2239 .await
2240 .is_err());
2241
2242 let consumer_b =
2244 create_v2_auth_context("consumer:b", "account:42", 1, limits, account_limits);
2245 assert!(manager
2246 .check_connection_allowed(addr, &Some(consumer_b))
2247 .await
2248 .is_ok());
2249 }
2250
2251 #[tokio::test]
2252 async fn policy_version_upgrade_stale_and_conflict_rules_apply() {
2253 let manager = ClientManager::new();
2254 let addr = create_test_socket_addr("127.0.0.1");
2255 let limits_v1 = Limits {
2256 max_connections: Some(5),
2257 ..Limits::default()
2258 };
2259 let limits_v2 = Limits {
2260 max_connections: Some(2),
2261 ..Limits::default()
2262 };
2263
2264 let v1 = create_v2_auth_context(
2266 "consumer:a",
2267 "account:42",
2268 1,
2269 Limits::default(),
2270 limits_v1.clone(),
2271 );
2272 assert!(manager
2273 .check_connection_allowed(addr, &Some(v1.clone()))
2274 .await
2275 .is_ok());
2276
2277 let v2 = create_v2_auth_context(
2279 "consumer:a",
2280 "account:42",
2281 2,
2282 Limits::default(),
2283 limits_v2.clone(),
2284 );
2285 assert!(manager
2286 .check_connection_allowed(addr, &Some(v2))
2287 .await
2288 .is_ok());
2289
2290 let stale = manager.check_connection_allowed(addr, &Some(v1)).await;
2292 assert!(stale.is_err());
2293
2294 let conflict =
2296 create_v2_auth_context("consumer:a", "account:42", 2, Limits::default(), limits_v1);
2297 assert!(manager
2298 .check_connection_allowed(addr, &Some(conflict))
2299 .await
2300 .is_err());
2301 }
2302
2303 #[tokio::test]
2304 async fn legacy_tokens_keep_old_behavior_and_are_counted() {
2305 let manager = ClientManager::new();
2306 let addr = create_test_socket_addr("127.0.0.1");
2307 let legacy = create_test_auth_context(
2308 "user-1",
2309 Limits {
2310 max_connections: Some(2),
2311 ..Limits::default()
2312 },
2313 );
2314 assert!(legacy.is_legacy_policy());
2315
2316 assert!(manager
2317 .check_connection_allowed(addr, &Some(legacy.clone()))
2318 .await
2319 .is_ok());
2320 assert_eq!(manager.account_policies().legacy_token_count(), 1);
2321 assert_eq!(manager.account_policies().tracked_accounts(), 0);
2323
2324 insert_client(&manager, legacy.clone());
2326 insert_client(&manager, legacy.clone());
2327 assert!(manager
2328 .check_connection_allowed(addr, &Some(legacy))
2329 .await
2330 .is_err());
2331 }
2332
2333 #[tokio::test]
2334 async fn account_message_budget_is_shared_across_consumers() {
2335 let manager = ClientManager::new();
2336 let account_limits = Limits {
2337 max_messages_per_minute: Some(2),
2338 ..Limits::default()
2339 };
2340 let client_a = insert_client(
2341 &manager,
2342 create_v2_auth_context(
2343 "consumer:a",
2344 "account:42",
2345 1,
2346 Limits::default(),
2347 account_limits.clone(),
2348 ),
2349 );
2350 let client_b = insert_client(
2351 &manager,
2352 create_v2_auth_context(
2353 "consumer:b",
2354 "account:42",
2355 1,
2356 Limits::default(),
2357 account_limits,
2358 ),
2359 );
2360
2361 assert!(manager.check_inbound_message_allowed(client_a).is_ok());
2362 assert!(manager.check_inbound_message_allowed(client_b).is_ok());
2363 assert!(manager.check_inbound_message_allowed(client_a).is_err());
2365 }
2366
2367 #[tokio::test]
2368 async fn account_state_evicts_when_idle_and_unreferenced() {
2369 let manager = ClientManager::new();
2370 let addr = create_test_socket_addr("127.0.0.1");
2371 let context = create_v2_auth_context(
2372 "consumer:a",
2373 "account:42",
2374 1,
2375 Limits::default(),
2376 Limits::default(),
2377 );
2378 assert!(manager
2379 .check_connection_allowed(addr, &Some(context))
2380 .await
2381 .is_ok());
2382 assert_eq!(manager.account_policies().tracked_accounts(), 1);
2383
2384 manager.account_policies().evict_idle(|_| false);
2387 assert_eq!(
2388 manager.account_policies().tracked_accounts(),
2389 1,
2390 "TTL keeps fresh entries"
2391 );
2392 manager.cleanup_account_state();
2393 assert_eq!(
2394 manager.account_policies().tracked_accounts(),
2395 1,
2396 "fresh entries survive sweep"
2397 );
2398 }
2399
2400 #[test]
2401 fn expired_token_is_removed_without_deadlocking() {
2402 let manager = ClientManager::new();
2403 let mut context = create_test_auth_context("user-1", Limits::default());
2404 context.expires_at = 1;
2405 let client_id = insert_client(&manager, context);
2406
2407 let (done_tx, done_rx) = std::sync::mpsc::channel();
2410 let sender = manager.clone();
2411 std::thread::spawn(move || {
2412 let result = sender.send_to_client(client_id, Arc::new(Bytes::from_static(b"update")));
2413 let _ = done_tx.send(result);
2414 });
2415
2416 let result = done_rx
2417 .recv_timeout(Duration::from_secs(5))
2418 .expect("send_to_client deadlocked on an expired token");
2419 assert_eq!(result, Err(SendError::ClientDisconnected));
2420 assert!(!manager.has_client(client_id));
2421 }
2422
2423 #[tokio::test]
2424 async fn an_expired_client_keeps_its_close_reason_when_its_queue_is_full() {
2425 let manager = ClientManager::new();
2426 let (sender, mut queue) = mpsc::channel(1);
2427 sender
2428 .try_send(Message::Text("backlog".into()))
2429 .expect("room for one message");
2430 let client_id = Uuid::new_v4();
2431 let mut context = create_test_auth_context("user-1", Limits::default());
2432 context.expires_at = 1;
2433 let client = ClientInfo::new(
2434 client_id,
2435 sender,
2436 Some(context),
2437 create_test_socket_addr("127.0.0.1"),
2438 );
2439 let close_frame = client.close_frame.clone();
2440 manager.clients.insert(client_id, client);
2441
2442 assert!(manager.check_and_remove_expired(client_id));
2443
2444 let frame = close_frame
2445 .get()
2446 .expect("the close reason is recorded even though the queue is full");
2447 assert_eq!(frame.code, CloseCode::Policy);
2448 assert_eq!(
2449 frame.reason.as_str(),
2450 "token-expired: Authentication token expired"
2451 );
2452 assert!(matches!(queue.recv().await, Some(Message::Text(_))));
2456 assert!(queue.recv().await.is_none());
2457 }
2458
2459 #[test]
2460 fn unexpired_token_is_kept() {
2461 let manager = ClientManager::new();
2462 let client_id = insert_client(
2463 &manager,
2464 create_test_auth_context("user-1", Limits::default()),
2465 );
2466
2467 assert!(!manager.check_and_remove_expired(client_id));
2468 assert!(manager.has_client(client_id));
2469 }
2470
2471 #[tokio::test]
2473 async fn test_rate_limiter_integration() {
2474 use crate::websocket::rate_limiter::{RateLimiterConfig, WebSocketRateLimiter};
2475
2476 let rate_limiter = Arc::new(WebSocketRateLimiter::new(RateLimiterConfig::default()));
2477 let manager = ClientManager::new().with_rate_limiter(rate_limiter);
2478 let addr = create_test_socket_addr("127.0.0.1");
2479
2480 let auth = create_test_auth_context("user-1", Limits::default());
2482 assert!(manager
2483 .check_connection_allowed(addr, &Some(auth))
2484 .await
2485 .is_ok());
2486 }
2487}