Skip to main content

arete_server/websocket/
client_manager.rs

1use crate::account_policy::{redact_identity, AccountPolicyError, AccountPolicyRegistry};
2use crate::compression::CompressedPayload;
3use crate::websocket::admission::{WebSocketAdmissionProvider, WebSocketConnectionPermit};
4use crate::websocket::auth::{AuthContext, AuthDeny, AuthErrorCode};
5use crate::websocket::rate_limiter::{RateLimitResult, WebSocketRateLimiter};
6use arete_auth::Limits;
7use bytes::Bytes;
8use dashmap::DashMap;
9use futures_util::stream::SplitSink;
10use futures_util::SinkExt;
11use std::collections::{HashMap, HashSet};
12use std::net::SocketAddr;
13use std::sync::Arc;
14use std::time::{Duration, Instant, SystemTime};
15use tokio::net::TcpStream;
16use tokio::sync::{mpsc, Mutex as AsyncMutex, RwLock};
17use tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode;
18use tokio_tungstenite::tungstenite::protocol::CloseFrame;
19use tokio_tungstenite::{tungstenite::Message, WebSocketStream};
20use tokio_util::sync::CancellationToken;
21use tracing::{debug, info, warn};
22use uuid::Uuid;
23
24pub type WebSocketSender = SplitSink<WebSocketStream<TcpStream>, Message>;
25
26/// Error type for send operations
27#[derive(Debug, Clone, Copy, PartialEq, Eq)]
28pub enum SendError {
29    /// Client not found in registry
30    ClientNotFound,
31    /// Client's message queue is full - client was disconnected
32    ClientBackpressured,
33    /// Client's channel is closed - client was disconnected
34    ClientDisconnected,
35}
36
37impl std::fmt::Display for SendError {
38    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
39        match self {
40            SendError::ClientNotFound => write!(f, "client not found"),
41            SendError::ClientBackpressured => write!(f, "client backpressured and disconnected"),
42            SendError::ClientDisconnected => write!(f, "client disconnected"),
43        }
44    }
45}
46
47impl std::error::Error for SendError {}
48
49/// Egress tracking for a client
50#[derive(Debug)]
51struct EgressTracker {
52    /// Bytes sent in the current minute window
53    bytes_this_minute: u64,
54    /// Start of the current minute window
55    window_start: SystemTime,
56}
57
58/// Inbound message-rate tracking for a client
59#[derive(Debug)]
60struct MessageRateTracker {
61    messages_this_minute: u32,
62    window_start: SystemTime,
63}
64
65impl MessageRateTracker {
66    fn new() -> Self {
67        Self {
68            messages_this_minute: 0,
69            window_start: SystemTime::now(),
70        }
71    }
72
73    fn maybe_reset_window(&mut self) {
74        let now = SystemTime::now();
75        if now.duration_since(self.window_start).unwrap_or_default() >= Duration::from_secs(60) {
76            self.messages_this_minute = 0;
77            self.window_start = now;
78        }
79    }
80
81    fn record_message(&mut self, limit: u32) -> bool {
82        self.maybe_reset_window();
83        if self.messages_this_minute + 1 > limit {
84            false
85        } else {
86            self.messages_this_minute += 1;
87            true
88        }
89    }
90
91    fn current_usage(&mut self) -> u32 {
92        self.maybe_reset_window();
93        self.messages_this_minute
94    }
95}
96
97impl EgressTracker {
98    fn new() -> Self {
99        Self {
100            bytes_this_minute: 0,
101            window_start: SystemTime::now(),
102        }
103    }
104
105    /// Check if we need to reset the window (new minute)
106    fn maybe_reset_window(&mut self) {
107        let now = SystemTime::now();
108        if now.duration_since(self.window_start).unwrap_or_default() >= Duration::from_secs(60) {
109            self.bytes_this_minute = 0;
110            self.window_start = now;
111        }
112    }
113
114    /// Record bytes sent, returning true if within limit
115    fn record_bytes(&mut self, bytes: usize, limit: u64) -> bool {
116        self.maybe_reset_window();
117        let bytes_u64 = bytes as u64;
118        if self.bytes_this_minute + bytes_u64 > limit {
119            false
120        } else {
121            self.bytes_this_minute += bytes_u64;
122            true
123        }
124    }
125
126    /// Get current usage
127    fn current_usage(&mut self) -> u64 {
128        self.maybe_reset_window();
129        self.bytes_this_minute
130    }
131}
132
133/// Aggregate message/egress usage shared by every connection of one account.
134///
135/// This is a per-process tracker, not a globally exact quota; durable
136/// cross-replica metering is a separate system.
137#[derive(Debug)]
138struct AccountUsage {
139    egress: std::sync::Mutex<EgressTracker>,
140    messages: std::sync::Mutex<MessageRateTracker>,
141    last_seen: std::sync::Mutex<Instant>,
142}
143
144impl AccountUsage {
145    fn new() -> Self {
146        Self {
147            egress: std::sync::Mutex::new(EgressTracker::new()),
148            messages: std::sync::Mutex::new(MessageRateTracker::new()),
149            last_seen: std::sync::Mutex::new(Instant::now()),
150        }
151    }
152
153    fn touch(&self) {
154        if let Ok(mut last_seen) = self.last_seen.lock() {
155            *last_seen = Instant::now();
156        }
157    }
158
159    fn record_egress(&self, bytes: usize, limit: u64) -> bool {
160        self.touch();
161        match self.egress.lock() {
162            Ok(mut tracker) => tracker.record_bytes(bytes, limit),
163            Err(_) => true,
164        }
165    }
166
167    fn record_message(&self, limit: u32) -> bool {
168        self.touch();
169        match self.messages.lock() {
170            Ok(mut tracker) => tracker.record_message(limit),
171            Err(_) => true,
172        }
173    }
174
175    fn idle_for(&self) -> Duration {
176        self.last_seen
177            .lock()
178            .map(|last_seen| last_seen.elapsed())
179            .unwrap_or(Duration::MAX)
180    }
181}
182
183/// Information about a connected client
184#[derive(Debug)]
185pub struct ClientInfo {
186    pub id: Uuid,
187    pub last_seen: SystemTime,
188    pub sender: mpsc::Sender<Message>,
189    subscriptions: Arc<RwLock<HashMap<String, CancellationToken>>>,
190    /// Authentication context for this client
191    pub auth_context: Option<AuthContext>,
192    /// Client's IP address for rate limiting
193    pub remote_addr: SocketAddr,
194    /// Egress tracking for rate limiting
195    egress_tracker: std::sync::Mutex<EgressTracker>,
196    /// Inbound message-rate tracking for rate limiting
197    message_rate_tracker: std::sync::Mutex<MessageRateTracker>,
198    /// Why the server dropped this client, sent as the close frame once the
199    /// queue drains. It is kept out of the queue so that a full queue cannot
200    /// lose it.
201    close_frame: Arc<std::sync::OnceLock<CloseFrame>>,
202    admission_permit: Option<Arc<dyn WebSocketConnectionPermit>>,
203    /// Serializes permit refreshes with subscription lifecycle callbacks for
204    /// this connection. Host callbacks run without a client-registry guard.
205    admission_lifecycle: Arc<AsyncMutex<()>>,
206}
207
208impl ClientInfo {
209    pub fn new(
210        id: Uuid,
211        sender: mpsc::Sender<Message>,
212        auth_context: Option<AuthContext>,
213        remote_addr: SocketAddr,
214    ) -> Self {
215        Self {
216            id,
217            last_seen: SystemTime::now(),
218            sender,
219            subscriptions: Arc::new(RwLock::new(HashMap::new())),
220            auth_context,
221            remote_addr,
222            egress_tracker: std::sync::Mutex::new(EgressTracker::new()),
223            message_rate_tracker: std::sync::Mutex::new(MessageRateTracker::new()),
224            close_frame: Arc::new(std::sync::OnceLock::new()),
225            admission_permit: None,
226            admission_lifecycle: Arc::new(AsyncMutex::new(())),
227        }
228    }
229
230    /// Record bytes sent, returning true if within limit
231    pub fn record_egress(&self, bytes: usize) -> Option<u64> {
232        if let Ok(mut tracker) = self.egress_tracker.lock() {
233            if let Some(ref ctx) = self.auth_context {
234                if let Some(limit) = ctx.limits.max_bytes_per_minute {
235                    if tracker.record_bytes(bytes, limit) {
236                        return Some(tracker.current_usage());
237                    } else {
238                        return None; // Limit exceeded
239                    }
240                }
241            }
242            // No limit set, return current usage (0)
243            return Some(tracker.current_usage());
244        }
245        None
246    }
247
248    /// Record an inbound client message, returning true if within limit.
249    pub fn record_inbound_message(&self) -> Option<u32> {
250        if let Ok(mut tracker) = self.message_rate_tracker.lock() {
251            if let Some(ref ctx) = self.auth_context {
252                if let Some(limit) = ctx.limits.max_messages_per_minute {
253                    if tracker.record_message(limit) {
254                        return Some(tracker.current_usage());
255                    } else {
256                        return None;
257                    }
258                }
259            }
260
261            return Some(tracker.current_usage());
262        }
263
264        None
265    }
266
267    pub fn update_last_seen(&mut self) {
268        self.last_seen = SystemTime::now();
269    }
270
271    pub fn is_stale(&self, timeout: Duration) -> bool {
272        self.last_seen.elapsed().unwrap_or(Duration::MAX) > timeout
273    }
274
275    pub async fn add_subscription(
276        &self,
277        subscription_id: String,
278        token: CancellationToken,
279    ) -> bool {
280        let _lifecycle = self.admission_lifecycle.lock().await;
281        let mut subs = self.subscriptions.write().await;
282        match subs.entry(subscription_id) {
283            std::collections::hash_map::Entry::Vacant(entry) => {
284                entry.insert(token);
285                true
286            }
287            std::collections::hash_map::Entry::Occupied(_) => false,
288        }
289    }
290
291    pub async fn remove_subscription(&self, subscription_id: &str) -> bool {
292        let _lifecycle = self.admission_lifecycle.lock().await;
293        let mut subs = self.subscriptions.write().await;
294        if let Some(token) = subs.remove(subscription_id) {
295            token.cancel();
296            debug!("Cancelled subscription: {}", subscription_id);
297            true
298        } else {
299            debug!(
300                "Subscription not found for cancellation: {}",
301                subscription_id
302            );
303            false
304        }
305    }
306
307    pub async fn cancel_all_subscriptions(&self) {
308        let _lifecycle = self.admission_lifecycle.lock().await;
309        let subs = self.subscriptions.read().await;
310        for (subscription_id, token) in subs.iter() {
311            token.cancel();
312            debug!("Cancelled subscription on disconnect: {}", subscription_id);
313        }
314    }
315
316    pub async fn subscription_count(&self) -> usize {
317        self.subscriptions.read().await.len()
318    }
319}
320
321/// Configuration for rate limiting in ClientManager
322///
323/// These settings control various rate limits at the connection level.
324/// Per-subject limits are controlled via AuthContext.Limits.
325#[derive(Debug, Clone)]
326pub struct RateLimitConfig {
327    /// Global maximum connections per IP address
328    pub max_connections_per_ip: Option<usize>,
329    /// Global maximum connections per metering key
330    pub max_connections_per_metering_key: Option<usize>,
331    /// Global maximum connections per origin
332    pub max_connections_per_origin: Option<usize>,
333    /// Default connection timeout for stale client cleanup
334    pub client_timeout: Duration,
335    /// Message queue size per client
336    pub message_queue_size: usize,
337    /// Maximum reconnect attempts per client (optional global default)
338    pub max_reconnect_attempts: Option<u32>,
339    /// Rate limit window duration for message counting
340    pub message_rate_window: Duration,
341    /// Rate limit window duration for egress tracking
342    pub egress_rate_window: Duration,
343    /// Default limits applied when auth token doesn't specify limits
344    /// These act as server-wide fallback limits for all connections
345    pub default_limits: Option<Limits>,
346}
347
348impl Default for RateLimitConfig {
349    fn default() -> Self {
350        Self {
351            max_connections_per_ip: None,
352            max_connections_per_metering_key: None,
353            max_connections_per_origin: None,
354            client_timeout: Duration::from_secs(300),
355            message_queue_size: 512,
356            max_reconnect_attempts: None,
357            message_rate_window: Duration::from_secs(60),
358            egress_rate_window: Duration::from_secs(60),
359            default_limits: None,
360        }
361    }
362}
363
364impl RateLimitConfig {
365    /// Load configuration from environment variables
366    ///
367    /// Environment variables:
368    /// - `ARETE_WS_MAX_CONNECTIONS_PER_IP` - Max connections per IP (default: unlimited)
369    /// - `ARETE_WS_MAX_CONNECTIONS_PER_METERING_KEY` - Max connections per metering key (default: unlimited)
370    /// - `ARETE_WS_MAX_CONNECTIONS_PER_ORIGIN` - Max connections per origin (default: unlimited)
371    /// - `ARETE_WS_CLIENT_TIMEOUT_SECS` - Client timeout in seconds (default: 300)
372    /// - `ARETE_WS_MESSAGE_QUEUE_SIZE` - Message queue size per client (default: 512)
373    /// - `ARETE_WS_RATE_LIMIT_WINDOW_SECS` - Rate limit window in seconds (default: 60)
374    /// - `ARETE_WS_DEFAULT_MAX_CONNECTIONS` - Default max connections per subject (fallback when token has no limit)
375    /// - `ARETE_WS_DEFAULT_MAX_SUBSCRIPTIONS` - Default max subscriptions per connection (fallback when token has no limit)
376    /// - `ARETE_WS_DEFAULT_MAX_SNAPSHOT_ROWS` - Default max snapshot rows per request (fallback when token has no limit)
377    /// - `ARETE_WS_DEFAULT_MAX_MESSAGES_PER_MINUTE` - Default max messages per minute (fallback when token has no limit)
378    /// - `ARETE_WS_DEFAULT_MAX_BYTES_PER_MINUTE` - Default max bytes per minute (fallback when token has no limit)
379    pub fn from_env() -> Self {
380        let mut config = Self::default();
381
382        if let Ok(val) = std::env::var("ARETE_WS_MAX_CONNECTIONS_PER_IP") {
383            if let Ok(max) = val.parse() {
384                config.max_connections_per_ip = Some(max);
385            }
386        }
387
388        if let Ok(val) = std::env::var("ARETE_WS_MAX_CONNECTIONS_PER_METERING_KEY") {
389            if let Ok(max) = val.parse() {
390                config.max_connections_per_metering_key = Some(max);
391            }
392        }
393
394        if let Ok(val) = std::env::var("ARETE_WS_MAX_CONNECTIONS_PER_ORIGIN") {
395            if let Ok(max) = val.parse() {
396                config.max_connections_per_origin = Some(max);
397            }
398        }
399
400        if let Ok(val) = std::env::var("ARETE_WS_CLIENT_TIMEOUT_SECS") {
401            if let Ok(secs) = val.parse() {
402                config.client_timeout = Duration::from_secs(secs);
403            }
404        }
405
406        if let Ok(val) = std::env::var("ARETE_WS_MESSAGE_QUEUE_SIZE") {
407            if let Ok(size) = val.parse() {
408                config.message_queue_size = size;
409            }
410        }
411
412        if let Ok(val) = std::env::var("ARETE_WS_RATE_LIMIT_WINDOW_SECS") {
413            if let Ok(secs) = val.parse() {
414                config.message_rate_window = Duration::from_secs(secs);
415                config.egress_rate_window = Duration::from_secs(secs);
416            }
417        }
418
419        // Load default limits from environment (fallback when auth token doesn't specify limits)
420        let mut default_limits = Limits::default();
421        let mut has_default_limits = false;
422
423        if let Ok(val) = std::env::var("ARETE_WS_DEFAULT_MAX_CONNECTIONS") {
424            if let Ok(max) = val.parse() {
425                default_limits.max_connections = Some(max);
426                has_default_limits = true;
427            }
428        }
429
430        if let Ok(val) = std::env::var("ARETE_WS_DEFAULT_MAX_SUBSCRIPTIONS") {
431            if let Ok(max) = val.parse() {
432                default_limits.max_subscriptions = Some(max);
433                has_default_limits = true;
434            }
435        }
436
437        if let Ok(val) = std::env::var("ARETE_WS_DEFAULT_MAX_SNAPSHOT_ROWS") {
438            if let Ok(max) = val.parse() {
439                default_limits.max_snapshot_rows = Some(max);
440                has_default_limits = true;
441            }
442        }
443
444        if let Ok(val) = std::env::var("ARETE_WS_DEFAULT_MAX_MESSAGES_PER_MINUTE") {
445            if let Ok(max) = val.parse() {
446                default_limits.max_messages_per_minute = Some(max);
447                has_default_limits = true;
448            }
449        }
450
451        if let Ok(val) = std::env::var("ARETE_WS_DEFAULT_MAX_BYTES_PER_MINUTE") {
452            if let Ok(max) = val.parse() {
453                default_limits.max_bytes_per_minute = Some(max);
454                has_default_limits = true;
455            }
456        }
457
458        if has_default_limits {
459            config.default_limits = Some(default_limits);
460        }
461
462        config
463    }
464
465    /// Set the maximum connections per IP
466    pub fn with_max_connections_per_ip(mut self, max: usize) -> Self {
467        self.max_connections_per_ip = Some(max);
468        self
469    }
470
471    /// Set the client timeout
472    pub fn with_timeout(mut self, timeout: Duration) -> Self {
473        self.client_timeout = timeout;
474        self
475    }
476
477    /// Set the message queue size
478    pub fn with_message_queue_size(mut self, size: usize) -> Self {
479        self.message_queue_size = size;
480        self
481    }
482
483    /// Set the rate limit window (applies to both message and egress windows)
484    pub fn with_rate_limit_window(mut self, window: Duration) -> Self {
485        self.message_rate_window = window;
486        self.egress_rate_window = window;
487        self
488    }
489
490    /// Set default limits applied when auth token doesn't specify limits
491    ///
492    /// These limits act as server-wide fallbacks for connections
493    /// where the authentication token doesn't include explicit limits.
494    pub fn with_default_limits(mut self, limits: Limits) -> Self {
495        self.default_limits = Some(limits);
496        self
497    }
498}
499
500/// Bound on tracked account usage entries; signed account keys keep the
501/// cardinality legitimate, this is a hard backstop.
502const MAX_TRACKED_ACCOUNT_USAGE: usize = crate::account_policy::DEFAULT_MAX_TRACKED_ACCOUNTS;
503
504/// Idle TTL after which aggregate account usage state is evicted.
505const ACCOUNT_USAGE_IDLE_TTL: Duration = crate::account_policy::DEFAULT_ACCOUNT_IDLE_TTL;
506
507/// Manages all connected WebSocket clients using lock-free DashMap.
508///
509/// Key design decisions:
510/// - Uses DashMap for lock-free concurrent access to client registry
511/// - Uses try_send instead of send to never block on slow clients
512/// - Disconnects clients that are backpressured (queue full) to prevent cascade failures
513/// - All public methods are non-blocking or use fine-grained per-key locks
514/// - Supports configurable rate limiting per IP, subject, and global defaults
515#[derive(Clone)]
516pub struct ClientManager {
517    clients: Arc<DashMap<Uuid, ClientInfo>>,
518    rate_limit_config: RateLimitConfig,
519    /// Optional WebSocket rate limiter for granular rate control
520    rate_limiter: Option<Arc<WebSocketRateLimiter>>,
521    /// Per-process account policy versions and signed aggregate limits
522    account_policies: Arc<AccountPolicyRegistry>,
523    /// Per-process aggregate message/egress usage per account
524    account_usage: Arc<DashMap<String, AccountUsage>>,
525    /// Optional embedder-owned admission state shared across runtimes.
526    admission_provider: Option<Arc<dyn WebSocketAdmissionProvider>>,
527}
528
529impl ClientManager {
530    pub fn new() -> Self {
531        Self::with_config(RateLimitConfig::default())
532    }
533
534    /// Create a new ClientManager with the given rate limit configuration
535    pub fn with_config(config: RateLimitConfig) -> Self {
536        Self {
537            clients: Arc::new(DashMap::new()),
538            rate_limit_config: config,
539            rate_limiter: None,
540            account_policies: Arc::new(AccountPolicyRegistry::default()),
541            account_usage: Arc::new(DashMap::new()),
542            admission_provider: None,
543        }
544    }
545
546    /// Install an embedder-owned provider for cross-runtime admission.
547    pub fn with_admission_provider(
548        mut self,
549        provider: Arc<dyn WebSocketAdmissionProvider>,
550    ) -> Self {
551        self.admission_provider = Some(provider);
552        self
553    }
554
555    #[allow(clippy::result_large_err)]
556    pub(crate) fn reserve_connection_admission(
557        &self,
558        context: &AuthContext,
559    ) -> Result<Option<Arc<dyn WebSocketConnectionPermit>>, AuthDeny> {
560        self.admission_provider
561            .as_ref()
562            .map(|provider| provider.reserve_connection(context))
563            .unwrap_or(Ok(None))
564    }
565
566    /// Load configuration from environment variables
567    ///
568    /// See `RateLimitConfig::from_env` for supported variables.
569    pub fn from_env() -> Self {
570        Self::with_config(RateLimitConfig::from_env())
571    }
572
573    /// Set the client timeout for stale client cleanup
574    pub fn with_timeout(mut self, timeout: Duration) -> Self {
575        self.rate_limit_config.client_timeout = timeout;
576        self
577    }
578
579    /// Set the message queue size per client
580    pub fn with_message_queue_size(mut self, queue_size: usize) -> Self {
581        self.rate_limit_config.message_queue_size = queue_size;
582        self
583    }
584
585    /// Set a global limit on connections per IP address
586    pub fn with_max_connections_per_ip(mut self, max: usize) -> Self {
587        self.rate_limit_config.max_connections_per_ip = Some(max);
588        self
589    }
590
591    /// Set the rate limit window duration
592    pub fn with_rate_limit_window(mut self, window: Duration) -> Self {
593        self.rate_limit_config.message_rate_window = window;
594        self.rate_limit_config.egress_rate_window = window;
595        self
596    }
597
598    /// Set default limits applied when auth token doesn't specify limits
599    ///
600    /// These limits act as server-wide fallbacks for connections
601    /// where the authentication token doesn't include explicit limits.
602    pub fn with_default_limits(mut self, limits: Limits) -> Self {
603        self.rate_limit_config.default_limits = Some(limits);
604        self
605    }
606
607    /// Set a WebSocket rate limiter for granular rate control
608    pub fn with_rate_limiter(mut self, rate_limiter: Arc<WebSocketRateLimiter>) -> Self {
609        self.rate_limiter = Some(rate_limiter);
610        self
611    }
612
613    /// Get the rate limiter if configured
614    pub fn rate_limiter(&self) -> Option<&WebSocketRateLimiter> {
615        self.rate_limiter.as_ref().map(|r| r.as_ref())
616    }
617
618    /// Get the current rate limit configuration
619    pub fn rate_limit_config(&self) -> &RateLimitConfig {
620        &self.rate_limit_config
621    }
622
623    /// Account policy state observed by this manager.
624    pub fn account_policies(&self) -> &AccountPolicyRegistry {
625        &self.account_policies
626    }
627
628    /// Record aggregate account egress; true when within the signed limit.
629    fn record_account_egress(&self, ctx: &AuthContext, bytes: usize) -> bool {
630        if ctx.is_legacy_policy() || ctx.account_key.is_none() {
631            return true;
632        }
633        let Some(limit) = ctx.account_limits.max_bytes_per_minute else {
634            return true;
635        };
636        let account = ctx.account_key();
637        if let Some(usage) = self.account_usage.get(account) {
638            return usage.record_egress(bytes, limit);
639        }
640        if self.account_usage.len() >= MAX_TRACKED_ACCOUNT_USAGE {
641            warn!(
642                account = %redact_identity(account),
643                "account usage tracking at capacity; aggregate egress guard skipped"
644            );
645            return true;
646        }
647        self.account_usage
648            .entry(account.to_string())
649            .or_insert_with(AccountUsage::new)
650            .record_egress(bytes, limit)
651    }
652
653    /// Record one aggregate account inbound message; true when within limit.
654    fn record_account_message(&self, ctx: &AuthContext) -> bool {
655        if ctx.is_legacy_policy() || ctx.account_key.is_none() {
656            return true;
657        }
658        let Some(limit) = ctx.account_limits.max_messages_per_minute else {
659            return true;
660        };
661        let account = ctx.account_key();
662        if let Some(usage) = self.account_usage.get(account) {
663            return usage.record_message(limit);
664        }
665        if self.account_usage.len() >= MAX_TRACKED_ACCOUNT_USAGE {
666            warn!(
667                account = %redact_identity(account),
668                "account usage tracking at capacity; aggregate message guard skipped"
669            );
670            return true;
671        }
672        self.account_usage
673            .entry(account.to_string())
674            .or_insert_with(AccountUsage::new)
675            .record_message(limit)
676    }
677
678    /// Enforce the per-connection and aggregate account egress budgets.
679    fn enforce_egress_budgets(&self, client_id: Uuid, bytes: usize) -> Result<(), SendError> {
680        let (connection_ok, account_ok, permit_id) = {
681            let Some(client) = self.clients.get(&client_id) else {
682                return Err(SendError::ClientNotFound);
683            };
684            let connection_ok = client.record_egress(bytes).is_some();
685            let account_ok = !connection_ok
686                || client
687                    .auth_context
688                    .as_ref()
689                    .map(|ctx| self.record_account_egress(ctx, bytes))
690                    .unwrap_or(true);
691            let permit_id = client.admission_permit.as_ref().map(|permit| permit.id());
692            (connection_ok, account_ok, permit_id)
693        };
694        if !connection_ok {
695            warn!("Client {} exceeded egress limit, disconnecting", client_id);
696            self.clients.remove(&client_id);
697            return Err(SendError::ClientDisconnected);
698        }
699        if !account_ok {
700            warn!(
701                "Client {} exceeded the account egress limit, disconnecting",
702                client_id
703            );
704            self.clients.remove(&client_id);
705            return Err(SendError::ClientDisconnected);
706        }
707        if let (Some(provider), Some(permit_id)) = (&self.admission_provider, permit_id) {
708            if let Err(deny) = provider.check_egress(permit_id, bytes) {
709                warn!(client = %client_id, reason = %deny.reason, "admission provider denied egress");
710                self.clients.remove(&client_id);
711                return Err(SendError::ClientDisconnected);
712            }
713        }
714        Ok(())
715    }
716
717    /// Add a new client connection.
718    ///
719    /// Spawns a dedicated sender task for this client that reads from its mpsc channel
720    /// and writes to the WebSocket. If the WebSocket write fails, the client is automatically
721    /// removed from the registry.
722    #[allow(clippy::result_large_err)]
723    pub fn add_client(
724        &self,
725        client_id: Uuid,
726        ws_sender: WebSocketSender,
727        auth_context: Option<AuthContext>,
728        remote_addr: SocketAddr,
729    ) -> Result<(), AuthDeny> {
730        let permit = if let Some(context) = auth_context.as_ref() {
731            self.reserve_connection_admission(context)?
732        } else {
733            None
734        };
735        self.add_client_with_admission(client_id, ws_sender, auth_context, remote_addr, permit);
736        Ok(())
737    }
738
739    pub(crate) fn add_client_with_admission(
740        &self,
741        client_id: Uuid,
742        mut ws_sender: WebSocketSender,
743        auth_context: Option<AuthContext>,
744        remote_addr: SocketAddr,
745        admission_permit: Option<Arc<dyn WebSocketConnectionPermit>>,
746    ) {
747        let (client_tx, mut client_rx) =
748            mpsc::channel::<Message>(self.rate_limit_config.message_queue_size);
749        let mut client_info = ClientInfo::new(client_id, client_tx, auth_context, remote_addr);
750        client_info.admission_permit = admission_permit;
751        let close_frame = client_info.close_frame.clone();
752
753        let clients_ref = self.clients.clone();
754        tokio::spawn(async move {
755            while let Some(message) = client_rx.recv().await {
756                if let Err(e) = ws_sender.send(message).await {
757                    warn!("Failed to send message to client {}: {}", client_id, e);
758                    break;
759                }
760            }
761            // Removing a client closes its queue. Complete the WebSocket close
762            // handshake as well so receivers do not wait forever on a socket
763            // whose server-side delivery task has already stopped, and say why
764            // when the server dropped the client on purpose.
765            if let Some(frame) = close_frame.get() {
766                let _ = ws_sender.send(Message::Close(Some(frame.clone()))).await;
767            }
768            let _ = ws_sender.close().await;
769            clients_ref.remove(&client_id);
770            debug!("WebSocket sender task for client {} stopped", client_id);
771        });
772
773        self.clients.insert(client_id, client_info);
774        info!("Client {} registered from {}", client_id, remote_addr);
775    }
776
777    /// Remove a client from the registry.
778    pub fn remove_client(&self, client_id: Uuid) {
779        if self.clients.remove(&client_id).is_some() {
780            info!("Client {} removed", client_id);
781        }
782    }
783
784    /// Update the auth context for a client.
785    ///
786    /// Used for in-band auth refresh without reconnecting.
787    pub fn update_client_auth(&self, client_id: Uuid, auth_context: AuthContext) -> bool {
788        self.try_update_client_auth(client_id, auth_context)
789            .unwrap_or(false)
790    }
791
792    /// Attempt a synchronous auth refresh without waiting for an in-flight
793    /// subscription admission change.
794    ///
795    /// Unlike [`Self::update_client_auth`], this preserves the reason a
796    /// refresh was not applied. Embedding hosts can retry an
797    /// [`AuthErrorCode::InternalError`] whose reason is `Subscription
798    /// admission is busy`, or use [`Self::update_client_auth_async`] when an
799    /// async call site can wait for the lifecycle lock.
800    #[allow(clippy::result_large_err)]
801    pub fn try_update_client_auth(
802        &self,
803        client_id: Uuid,
804        auth_context: AuthContext,
805    ) -> Result<bool, AuthDeny> {
806        let Some(lifecycle) = self
807            .clients
808            .get(&client_id)
809            .map(|client| client.admission_lifecycle.clone())
810        else {
811            return Ok(false);
812        };
813        let _lifecycle = lifecycle.try_lock().map_err(|_| {
814            AuthDeny::new(
815                AuthErrorCode::InternalError,
816                "Subscription admission is busy",
817            )
818        })?;
819
820        let (subscriptions, current_permit) = {
821            let Some(client) = self.clients.get(&client_id) else {
822                return Ok(false);
823            };
824            if !Arc::ptr_eq(&client.admission_lifecycle, &lifecycle) {
825                return Ok(false);
826            }
827            (
828                client.subscriptions.clone(),
829                client.admission_permit.clone(),
830            )
831        };
832        let subscriptions = subscriptions.try_read().map_err(|_| {
833            AuthDeny::new(
834                AuthErrorCode::InternalError,
835                "Subscription admission is busy",
836            )
837        })?;
838        let active_subscriptions = subscriptions.keys().cloned().collect::<Vec<_>>();
839        drop(subscriptions);
840
841        let permit = match &self.admission_provider {
842            Some(provider) => {
843                provider.refresh_connection(current_permit, &auth_context, &active_subscriptions)?
844            }
845            None => current_permit,
846        };
847
848        let Some(mut client) = self.clients.get_mut(&client_id) else {
849            return Ok(false);
850        };
851        if !Arc::ptr_eq(&client.admission_lifecycle, &lifecycle) {
852            return Ok(false);
853        }
854        client.admission_permit = permit;
855        client.auth_context = Some(auth_context);
856        debug!("Updated auth context for client {}", client_id);
857        Ok(true)
858    }
859
860    /// Wait for an in-flight subscription admission change before refreshing
861    /// authentication. The synchronous [`Self::update_client_auth`] remains
862    /// available for embedding hosts that need its original non-blocking API.
863    pub async fn update_client_auth_async(
864        &self,
865        client_id: Uuid,
866        auth_context: AuthContext,
867    ) -> bool {
868        self.try_update_client_auth_async(client_id, auth_context)
869            .await
870            .unwrap_or(false)
871    }
872
873    #[allow(clippy::result_large_err)]
874    pub(crate) async fn try_update_client_auth_async(
875        &self,
876        client_id: Uuid,
877        auth_context: AuthContext,
878    ) -> Result<bool, AuthDeny> {
879        let Some(lifecycle) = self
880            .clients
881            .get(&client_id)
882            .map(|client| client.admission_lifecycle.clone())
883        else {
884            return Ok(false);
885        };
886        let _lifecycle = lifecycle.lock().await;
887
888        let (subscriptions, current_permit) = {
889            let Some(client) = self.clients.get(&client_id) else {
890                return Ok(false);
891            };
892            if !Arc::ptr_eq(&client.admission_lifecycle, &lifecycle) {
893                return Ok(false);
894            }
895            (
896                client.subscriptions.clone(),
897                client.admission_permit.clone(),
898            )
899        };
900        let subscriptions = subscriptions.read().await;
901        let active_subscriptions = subscriptions.keys().cloned().collect::<Vec<_>>();
902        drop(subscriptions);
903
904        let permit = match &self.admission_provider {
905            Some(provider) => {
906                provider.refresh_connection(current_permit, &auth_context, &active_subscriptions)?
907            }
908            None => current_permit,
909        };
910
911        let Some(mut client) = self.clients.get_mut(&client_id) else {
912            return Ok(false);
913        };
914        if !Arc::ptr_eq(&client.admission_lifecycle, &lifecycle) {
915            return Ok(false);
916        }
917        client.admission_permit = permit;
918        client.auth_context = Some(auth_context);
919        debug!("Updated auth context for client {}", client_id);
920        Ok(true)
921    }
922
923    /// Check if a client's token has expired.
924    ///
925    /// Returns true if the client has an auth context and it has expired.
926    /// If expired, the client is removed from the registry.
927    pub fn check_and_remove_expired(&self, client_id: Uuid) -> bool {
928        let now = std::time::SystemTime::now()
929            .duration_since(std::time::UNIX_EPOCH)
930            .unwrap_or_default()
931            .as_secs();
932        // Check and remove under one shard lock. Calling `remove` while a
933        // `get` guard on the same key is alive deadlocks this thread, and
934        // every later caller that touches the shard queues behind it.
935        let Some((_, client)) = self.clients.remove_if(&client_id, |_, client| {
936            client
937                .auth_context
938                .as_ref()
939                .is_some_and(|ctx| ctx.expires_at <= now)
940        }) else {
941            return false;
942        };
943        if let Some(ctx) = &client.auth_context {
944            warn!(
945                "Client {} token expired (expired at {}), disconnecting",
946                client_id, ctx.expires_at
947            );
948        }
949        // Say why the socket closes. The sender task sends this after the
950        // queue drains, so it is the last frame the client sees; the SDKs read
951        // a `token-expired:` reason as "mint a new token and reconnect".
952        let _ = client.close_frame.set(CloseFrame {
953            code: CloseCode::Policy,
954            reason: format!(
955                "{}: Authentication token expired",
956                AuthErrorCode::TokenExpired.as_str()
957            )
958            .into(),
959        });
960        true
961    }
962
963    /// Get the current number of connected clients.
964    ///
965    /// This is lock-free and returns an approximate count (may be slightly stale
966    /// under high concurrency, which is fine for max_clients checks).
967    pub fn client_count(&self) -> usize {
968        self.clients.len()
969    }
970
971    /// Send data to a specific client (non-blocking).
972    ///
973    /// This method NEVER blocks. If the client's queue is full, the client is
974    /// considered too slow and is disconnected to prevent cascade failures.
975    /// Use this for live streaming updates.
976    ///
977    /// For initial snapshots where you expect to send many messages at once,
978    /// use `send_to_client_async` instead which will wait for queue space.
979    pub fn send_to_client(&self, client_id: Uuid, data: Arc<Bytes>) -> Result<(), SendError> {
980        // Check if client token has expired before sending
981        if self.check_and_remove_expired(client_id) {
982            return Err(SendError::ClientDisconnected);
983        }
984
985        // Check per-connection and aggregate account egress limits
986        self.enforce_egress_budgets(client_id, data.len())?;
987
988        let sender = {
989            let client = self
990                .clients
991                .get(&client_id)
992                .ok_or(SendError::ClientNotFound)?;
993            client.sender.clone()
994        };
995
996        let msg = Message::Binary((*data).clone());
997        match sender.try_send(msg) {
998            Ok(()) => Ok(()),
999            Err(mpsc::error::TrySendError::Full(_)) => {
1000                warn!(
1001                    "Client {} backpressured (queue full), disconnecting",
1002                    client_id
1003                );
1004                self.clients.remove(&client_id);
1005                Err(SendError::ClientBackpressured)
1006            }
1007            Err(mpsc::error::TrySendError::Closed(_)) => {
1008                debug!("Client {} channel closed", client_id);
1009                self.clients.remove(&client_id);
1010                Err(SendError::ClientDisconnected)
1011            }
1012        }
1013    }
1014
1015    /// Send data to a specific client (async, waits for queue space).
1016    ///
1017    /// This method will wait if the client's queue is full, allowing the client
1018    /// time to catch up. Use this for initial snapshots where you need to send
1019    /// many messages at once.
1020    ///
1021    /// For live streaming updates, use `send_to_client` instead which will
1022    /// disconnect slow clients rather than blocking.
1023    pub async fn send_to_client_async(
1024        &self,
1025        client_id: Uuid,
1026        data: Arc<Bytes>,
1027    ) -> Result<(), SendError> {
1028        // Check if client token has expired before sending
1029        if self.check_and_remove_expired(client_id) {
1030            return Err(SendError::ClientDisconnected);
1031        }
1032
1033        // Check per-connection and aggregate account egress limits
1034        self.enforce_egress_budgets(client_id, data.len())?;
1035
1036        let sender = {
1037            let client = self
1038                .clients
1039                .get(&client_id)
1040                .ok_or(SendError::ClientNotFound)?;
1041            client.sender.clone()
1042        };
1043
1044        let msg = Message::Binary((*data).clone());
1045        sender
1046            .send(msg)
1047            .await
1048            .map_err(|_| SendError::ClientDisconnected)
1049    }
1050
1051    /// Send a text message to a specific client (async).
1052    ///
1053    /// This method sends a text message directly to the client's WebSocket.
1054    /// Used for control messages like auth refresh responses.
1055    pub async fn send_text_to_client(
1056        &self,
1057        client_id: Uuid,
1058        text: String,
1059    ) -> Result<(), SendError> {
1060        // Check if client token has expired before sending
1061        if self.check_and_remove_expired(client_id) {
1062            return Err(SendError::ClientDisconnected);
1063        }
1064
1065        let sender = {
1066            let client = self
1067                .clients
1068                .get(&client_id)
1069                .ok_or(SendError::ClientNotFound)?;
1070            client.sender.clone()
1071        };
1072
1073        let msg = Message::Text(text.into());
1074        sender
1075            .send(msg)
1076            .await
1077            .map_err(|_| SendError::ClientDisconnected)
1078    }
1079
1080    /// Send a potentially compressed payload to a client (async).
1081    ///
1082    /// Compressed payloads are sent as binary frames (raw gzip).
1083    /// Uncompressed payloads are sent as text frames (JSON).
1084    pub async fn send_compressed_async(
1085        &self,
1086        client_id: Uuid,
1087        payload: CompressedPayload,
1088    ) -> Result<(), SendError> {
1089        // Check if client token has expired before sending
1090        if self.check_and_remove_expired(client_id) {
1091            return Err(SendError::ClientDisconnected);
1092        }
1093
1094        let (sender, bytes_to_record) = {
1095            let client = self
1096                .clients
1097                .get(&client_id)
1098                .ok_or(SendError::ClientNotFound)?;
1099
1100            let bytes = match &payload {
1101                CompressedPayload::Compressed(bytes) => bytes.len(),
1102                CompressedPayload::Uncompressed(bytes) => bytes.len(),
1103            };
1104
1105            (client.sender.clone(), bytes)
1106        };
1107
1108        // Check per-connection and aggregate account egress limits
1109        self.enforce_egress_budgets(client_id, bytes_to_record)?;
1110
1111        let msg = match payload {
1112            CompressedPayload::Compressed(bytes) => Message::Binary(bytes),
1113            CompressedPayload::Uncompressed(bytes) => Message::Binary(bytes),
1114        };
1115        sender
1116            .send(msg)
1117            .await
1118            .map_err(|_| SendError::ClientDisconnected)
1119    }
1120
1121    /// Update the last_seen timestamp for a client.
1122    pub fn update_client_last_seen(&self, client_id: Uuid) {
1123        if let Some(mut client) = self.clients.get_mut(&client_id) {
1124            client.update_last_seen();
1125        }
1126    }
1127
1128    /// Check whether an inbound message is allowed for a client.
1129    #[allow(clippy::result_large_err)]
1130    pub fn check_inbound_message_allowed(&self, client_id: Uuid) -> Result<(), AuthDeny> {
1131        if self.check_and_remove_expired(client_id) {
1132            return Err(AuthDeny::new(
1133                crate::websocket::auth::AuthErrorCode::TokenExpired,
1134                "Authentication token expired",
1135            ));
1136        }
1137
1138        let (connection_ok, account_ok, permit_id) = {
1139            let Some(client) = self.clients.get(&client_id) else {
1140                return Err(AuthDeny::new(
1141                    crate::websocket::auth::AuthErrorCode::InternalError,
1142                    "Client not found",
1143                ));
1144            };
1145            let connection_ok = client.record_inbound_message().is_some();
1146            let account_ok = !connection_ok
1147                || client
1148                    .auth_context
1149                    .as_ref()
1150                    .map(|ctx| self.record_account_message(ctx))
1151                    .unwrap_or(true);
1152            let permit_id = client.admission_permit.as_ref().map(|permit| permit.id());
1153            (connection_ok, account_ok, permit_id)
1154        };
1155
1156        if connection_ok && account_ok {
1157            if let (Some(provider), Some(permit_id)) = (&self.admission_provider, permit_id) {
1158                provider.check_inbound_message(permit_id)?;
1159            }
1160            return Ok(());
1161        }
1162        let scope = if connection_ok {
1163            "inbound account websocket messages"
1164        } else {
1165            "inbound websocket messages"
1166        };
1167        Err(
1168            AuthDeny::rate_limited(self.rate_limit_config.message_rate_window, scope).with_context(
1169                format!("client {} exceeded the inbound message budget", client_id),
1170            ),
1171        )
1172    }
1173
1174    /// Check if a client exists.
1175    pub fn has_client(&self, client_id: Uuid) -> bool {
1176        self.clients.contains_key(&client_id)
1177    }
1178
1179    pub async fn add_client_subscription(
1180        &self,
1181        client_id: Uuid,
1182        subscription_id: String,
1183        token: CancellationToken,
1184    ) -> bool {
1185        self.try_add_client_subscription(client_id, subscription_id, token)
1186            .await
1187            .unwrap_or(false)
1188    }
1189
1190    #[allow(clippy::result_large_err)]
1191    pub(crate) async fn try_add_client_subscription(
1192        &self,
1193        client_id: Uuid,
1194        subscription_id: String,
1195        token: CancellationToken,
1196    ) -> Result<bool, AuthDeny> {
1197        let Some(lifecycle) = self
1198            .clients
1199            .get(&client_id)
1200            .map(|client| client.admission_lifecycle.clone())
1201        else {
1202            return Ok(false);
1203        };
1204        let _lifecycle = lifecycle.lock().await;
1205        let Some((subscriptions, permit)) = self.clients.get(&client_id).and_then(|client| {
1206            Arc::ptr_eq(&client.admission_lifecycle, &lifecycle).then(|| {
1207                (
1208                    client.subscriptions.clone(),
1209                    client.admission_permit.clone(),
1210                )
1211            })
1212        }) else {
1213            return Ok(false);
1214        };
1215        let permit_id = permit.as_ref().map(|permit| permit.id());
1216        if let (Some(provider), Some(permit_id)) = (&self.admission_provider, permit_id) {
1217            if !provider.add_subscription(permit_id, &subscription_id)? {
1218                return Ok(false);
1219            }
1220        }
1221        let mut subscriptions = subscriptions.write().await;
1222        let client = self.clients.get(&client_id);
1223        let registered = client
1224            .as_ref()
1225            .is_some_and(|client| Arc::ptr_eq(&client.admission_lifecycle, &lifecycle));
1226        if !registered || subscriptions.contains_key(&subscription_id) {
1227            drop(client);
1228            drop(subscriptions);
1229            if let (Some(provider), Some(permit_id)) = (&self.admission_provider, permit_id) {
1230                provider.remove_subscription(permit_id, &subscription_id);
1231            }
1232            return Ok(false);
1233        }
1234        subscriptions.insert(subscription_id, token);
1235        drop(client);
1236        Ok(true)
1237    }
1238
1239    pub async fn remove_client_subscription(&self, client_id: Uuid, subscription_id: &str) -> bool {
1240        let Some(lifecycle) = self
1241            .clients
1242            .get(&client_id)
1243            .map(|client| client.admission_lifecycle.clone())
1244        else {
1245            return false;
1246        };
1247        let _lifecycle = lifecycle.lock().await;
1248        let Some((subscriptions, permit)) = self.clients.get(&client_id).and_then(|client| {
1249            Arc::ptr_eq(&client.admission_lifecycle, &lifecycle).then(|| {
1250                (
1251                    client.subscriptions.clone(),
1252                    client.admission_permit.clone(),
1253                )
1254            })
1255        }) else {
1256            return false;
1257        };
1258        let permit_id = permit.as_ref().map(|permit| permit.id());
1259        let mut subscriptions = subscriptions.write().await;
1260        let client = self.clients.get(&client_id);
1261        if !client
1262            .as_ref()
1263            .is_some_and(|client| Arc::ptr_eq(&client.admission_lifecycle, &lifecycle))
1264        {
1265            return false;
1266        }
1267        let token = subscriptions.remove(subscription_id);
1268        drop(client);
1269        drop(subscriptions);
1270        if let Some(token) = token {
1271            token.cancel();
1272            if let (Some(provider), Some(permit_id)) = (&self.admission_provider, permit_id) {
1273                provider.remove_subscription(permit_id, subscription_id);
1274            }
1275            true
1276        } else {
1277            false
1278        }
1279    }
1280
1281    /// Give an active subscription a new cancellation token, cancelling the
1282    /// one it had, without releasing its registration or admission: its
1283    /// delivery stops and the caller may attach a new one under the same ID.
1284    /// Returns `false` when the client or subscription is gone.
1285    pub async fn replace_client_subscription_token(
1286        &self,
1287        client_id: Uuid,
1288        subscription_id: &str,
1289        cancel_token: CancellationToken,
1290    ) -> bool {
1291        let Some(lifecycle) = self
1292            .clients
1293            .get(&client_id)
1294            .map(|client| client.admission_lifecycle.clone())
1295        else {
1296            return false;
1297        };
1298        let _lifecycle = lifecycle.lock().await;
1299        let Some(subscriptions) = self.clients.get(&client_id).and_then(|client| {
1300            Arc::ptr_eq(&client.admission_lifecycle, &lifecycle)
1301                .then(|| client.subscriptions.clone())
1302        }) else {
1303            return false;
1304        };
1305        let mut subscriptions = subscriptions.write().await;
1306        match subscriptions.get_mut(subscription_id) {
1307            Some(token) => {
1308                std::mem::replace(token, cancel_token).cancel();
1309                true
1310            }
1311            None => false,
1312        }
1313    }
1314
1315    pub async fn cancel_all_client_subscriptions(&self, client_id: Uuid) {
1316        let Some(lifecycle) = self
1317            .clients
1318            .get(&client_id)
1319            .map(|client| client.admission_lifecycle.clone())
1320        else {
1321            return;
1322        };
1323        let _lifecycle = lifecycle.lock().await;
1324        let Some(subscriptions) = self.clients.get(&client_id).and_then(|client| {
1325            Arc::ptr_eq(&client.admission_lifecycle, &lifecycle)
1326                .then(|| client.subscriptions.clone())
1327        }) else {
1328            return;
1329        };
1330        let subscriptions = subscriptions.read().await;
1331        for (subscription_id, token) in subscriptions.iter() {
1332            token.cancel();
1333            debug!("Cancelled subscription on disconnect: {}", subscription_id);
1334        }
1335    }
1336
1337    /// Remove stale clients that haven't been seen within the timeout period.
1338    pub fn cleanup_stale_clients(&self) -> usize {
1339        let timeout = self.rate_limit_config.client_timeout;
1340        let mut stale_clients = Vec::new();
1341
1342        for entry in self.clients.iter() {
1343            if entry.value().is_stale(timeout) {
1344                stale_clients.push(*entry.key());
1345            }
1346        }
1347
1348        let removed_count = stale_clients.len();
1349        for client_id in stale_clients {
1350            self.clients.remove(&client_id);
1351            info!("Removed stale client {}", client_id);
1352        }
1353
1354        removed_count
1355    }
1356
1357    /// Evict aggregate account usage and policy state for accounts with no
1358    /// live connection past the idle TTL.
1359    fn cleanup_account_state(&self) {
1360        let live_accounts: HashSet<String> = self
1361            .clients
1362            .iter()
1363            .filter_map(|entry| {
1364                entry
1365                    .value()
1366                    .auth_context
1367                    .as_ref()
1368                    .and_then(|ctx| ctx.account_key.clone())
1369            })
1370            .collect();
1371        self.account_usage.retain(|account, usage| {
1372            live_accounts.contains(account) || usage.idle_for() < ACCOUNT_USAGE_IDLE_TTL
1373        });
1374        self.account_policies
1375            .evict_idle(|account| live_accounts.contains(account));
1376    }
1377
1378    /// Start a background task that periodically cleans up stale clients.
1379    ///
1380    /// Returns the task so an owner that stops serving can abort it; callers
1381    /// that run for the life of the process may simply drop the handle.
1382    pub fn start_cleanup_task(&self) -> tokio::task::JoinHandle<()> {
1383        let client_manager = self.clone();
1384
1385        tokio::spawn(async move {
1386            let mut interval = tokio::time::interval(Duration::from_secs(30));
1387
1388            loop {
1389                interval.tick().await;
1390                let removed = client_manager.cleanup_stale_clients();
1391                if removed > 0 {
1392                    info!("Cleaned up {} stale clients", removed);
1393                }
1394                client_manager.cleanup_account_state();
1395                if let Some(provider) = &client_manager.admission_provider {
1396                    provider.cleanup();
1397                }
1398            }
1399        })
1400    }
1401
1402    /// ENFORCEMENT HOOKS
1403    ///
1404    /// These methods provide hooks for enforcing limits based on auth context.
1405    /// They check limits before allowing operations and return errors if limits are exceeded.
1406    /// Check if a connection is allowed for the given auth context.
1407    ///
1408    /// Returns Ok(()) if the connection is allowed, or an error with a reason if not.
1409    ///
1410    /// Publishable-key sessions get one fixed reason per refusal: the detailed
1411    /// reasons name accounts, consumers and connection counts.
1412    #[allow(clippy::result_large_err)]
1413    pub async fn check_connection_allowed(
1414        &self,
1415        remote_addr: SocketAddr,
1416        auth_context: &Option<AuthContext>,
1417    ) -> Result<(), AuthDeny> {
1418        let publishable = auth_context
1419            .as_ref()
1420            .is_some_and(|ctx| ctx.key_class == arete_auth::KeyClass::Publishable);
1421        self.check_connection_limits(remote_addr, auth_context)
1422            .await
1423            .map_err(|deny| {
1424                if publishable {
1425                    deny.for_publishable_session()
1426                } else {
1427                    deny
1428                }
1429            })
1430    }
1431
1432    #[allow(clippy::result_large_err)]
1433    async fn check_connection_limits(
1434        &self,
1435        remote_addr: SocketAddr,
1436        auth_context: &Option<AuthContext>,
1437    ) -> Result<(), AuthDeny> {
1438        // Check rate limiter first if configured
1439        if let Some(ref rate_limiter) = self.rate_limiter {
1440            // Check handshake rate limit for IP
1441            match rate_limiter.check_handshake(remote_addr).await {
1442                RateLimitResult::Allowed { .. } => {}
1443                RateLimitResult::Denied { retry_after, limit } => {
1444                    return Err(AuthDeny::rate_limited(retry_after, "websocket handshakes")
1445                        .with_context(format!(
1446                            "handshake rate limit of {} per minute exceeded for {}",
1447                            limit, remote_addr
1448                        )));
1449                }
1450            }
1451
1452            if let Some(ref ctx) = auth_context {
1453                // Connection-attempt rate per resolved consumer. Legacy
1454                // tokens resolve to the subject with the configured window,
1455                // preserving old behavior exactly.
1456                match rate_limiter
1457                    .check_connection_for_consumer(
1458                        ctx.consumer_key(),
1459                        ctx.limits.max_connection_attempts_per_minute,
1460                    )
1461                    .await
1462                {
1463                    RateLimitResult::Allowed { .. } => {}
1464                    RateLimitResult::Denied { retry_after, limit } => {
1465                        return Err(AuthDeny::rate_limited(retry_after, "websocket connections")
1466                            .with_context(format!(
1467                                "connection rate limit for consumer {} of {} per minute exceeded",
1468                                ctx.consumer_key(),
1469                                limit
1470                            )));
1471                    }
1472                }
1473
1474                // Connection-attempt rate per resolved account. Legacy
1475                // tokens resolve to the metering key; anonymous v2 tokens
1476                // carry no account and skip the aggregate bucket.
1477                if ctx.is_legacy_policy() || ctx.account_key.is_some() {
1478                    match rate_limiter
1479                        .check_connection_for_account(
1480                            ctx.account_key(),
1481                            ctx.account_limits.max_connection_attempts_per_minute,
1482                        )
1483                        .await
1484                    {
1485                        RateLimitResult::Allowed { .. } => {}
1486                        RateLimitResult::Denied { retry_after, limit } => {
1487                            return Err(AuthDeny::rate_limited(
1488                                retry_after,
1489                                "metered websocket connections",
1490                            )
1491                            .with_context(format!(
1492                                "connection rate limit for account {} of {} per minute exceeded",
1493                                ctx.account_key(),
1494                                limit
1495                            )));
1496                        }
1497                    }
1498                }
1499            }
1500        }
1501
1502        // Check global per-IP connection limit
1503        if let Some(max_per_ip) = self.rate_limit_config.max_connections_per_ip {
1504            let current_ip_connections = self.count_connections_for_ip(&remote_addr);
1505            if current_ip_connections >= max_per_ip {
1506                return Err(AuthDeny::connection_limit_exceeded(
1507                    &format!("ip {}", remote_addr.ip()),
1508                    current_ip_connections,
1509                    max_per_ip,
1510                ));
1511            }
1512        }
1513
1514        if let Some(ctx) = auth_context {
1515            // Admit the token against previously observed account policy and
1516            // count legacy tokens so legacy support can be retired once none remain.
1517            if ctx.is_legacy_policy() {
1518                let legacy_policy_token = self.account_policies.record_legacy_token();
1519                debug!(legacy_policy_token, "legacy policy token admitted");
1520            } else if let (Some(account), Some(policy_version)) =
1521                (ctx.account_key.as_deref(), ctx.policy_version)
1522            {
1523                match self
1524                    .account_policies
1525                    .observe(account, policy_version, &ctx.account_limits)
1526                {
1527                    Ok(()) => {}
1528                    Err(AccountPolicyError::StaleVersion { presented, current }) => {
1529                        debug!(
1530                            account = %redact_identity(account),
1531                            presented,
1532                            current,
1533                            "stale policy version rejected"
1534                        );
1535                        return Err(AuthDeny::new(
1536                            crate::websocket::auth::AuthErrorCode::TokenExpired,
1537                            "session policy version is stale; refresh the session token",
1538                        ));
1539                    }
1540                    Err(AccountPolicyError::ConflictingLimits { version }) => {
1541                        warn!(
1542                            account = %redact_identity(account),
1543                            version,
1544                            "signed account limits conflict for one policy version"
1545                        );
1546                        return Err(AuthDeny::new(
1547                            crate::websocket::auth::AuthErrorCode::InternalError,
1548                            "signed account limits conflict with previously observed policy",
1549                        ));
1550                    }
1551                    Err(AccountPolicyError::CapacityExhausted) => {
1552                        warn!("account policy state at capacity; denying admission");
1553                        return Err(AuthDeny::new(
1554                            crate::websocket::auth::AuthErrorCode::InternalError,
1555                            "account policy state is at capacity; retry shortly",
1556                        ));
1557                    }
1558                }
1559            }
1560
1561            // Check max connections per resolved consumer (token limits,
1562            // fallback to default limits). Legacy tokens resolve to the
1563            // subject, preserving old behavior.
1564            let max_connections = ctx.limits.max_connections.or_else(|| {
1565                self.rate_limit_config
1566                    .default_limits
1567                    .as_ref()
1568                    .and_then(|l| l.max_connections)
1569            });
1570            if let Some(max_connections) = max_connections {
1571                let current_connections = self.count_connections_for_consumer(ctx.consumer_key());
1572                if current_connections >= max_connections as usize {
1573                    return Err(AuthDeny::connection_limit_exceeded(
1574                        &format!("consumer {}", ctx.consumer_key()),
1575                        current_connections,
1576                        max_connections as usize,
1577                    ));
1578                }
1579            }
1580
1581            // Check aggregate concurrent connections per account from the
1582            // signed account limits.
1583            if !ctx.is_legacy_policy() && ctx.account_key.is_some() {
1584                if let Some(max_account_connections) = ctx.account_limits.max_connections {
1585                    let current_account_connections =
1586                        self.count_connections_for_account(ctx.account_key());
1587                    if current_account_connections >= max_account_connections as usize {
1588                        return Err(AuthDeny::connection_limit_exceeded(
1589                            &format!("account {}", ctx.account_key()),
1590                            current_account_connections,
1591                            max_account_connections as usize,
1592                        ));
1593                    }
1594                }
1595            }
1596
1597            // Check global max connections per metering key
1598            if let Some(max_per_metering_key) =
1599                self.rate_limit_config.max_connections_per_metering_key
1600            {
1601                let current_metering_connections =
1602                    self.count_connections_for_metering_key(&ctx.metering_key);
1603                if current_metering_connections >= max_per_metering_key {
1604                    return Err(AuthDeny::connection_limit_exceeded(
1605                        &format!("metering key {}", ctx.metering_key),
1606                        current_metering_connections,
1607                        max_per_metering_key,
1608                    ));
1609                }
1610            }
1611
1612            // Check global max connections per origin
1613            if let Some(max_per_origin) = self.rate_limit_config.max_connections_per_origin {
1614                if let Some(ref origin) = ctx.origin {
1615                    let current_origin_connections = self.count_connections_for_origin(origin);
1616                    if current_origin_connections >= max_per_origin {
1617                        return Err(AuthDeny::connection_limit_exceeded(
1618                            &format!("origin {}", origin),
1619                            current_origin_connections,
1620                            max_per_origin,
1621                        ));
1622                    }
1623                }
1624            }
1625        }
1626        Ok(())
1627    }
1628
1629    /// Count connections from a specific IP address
1630    fn count_connections_for_ip(&self, remote_addr: &SocketAddr) -> usize {
1631        let ip = remote_addr.ip();
1632        self.clients
1633            .iter()
1634            .filter(|entry| entry.value().remote_addr.ip() == ip)
1635            .count()
1636    }
1637
1638    /// Count connections for a resolved consumer identity
1639    fn count_connections_for_consumer(&self, consumer: &str) -> usize {
1640        self.clients
1641            .iter()
1642            .filter(|entry| {
1643                entry
1644                    .value()
1645                    .auth_context
1646                    .as_ref()
1647                    .map(|ctx| ctx.consumer_key() == consumer)
1648                    .unwrap_or(false)
1649            })
1650            .count()
1651    }
1652
1653    /// Count connections for a resolved account identity
1654    fn count_connections_for_account(&self, account: &str) -> usize {
1655        self.clients
1656            .iter()
1657            .filter(|entry| {
1658                entry
1659                    .value()
1660                    .auth_context
1661                    .as_ref()
1662                    .map(|ctx| ctx.account_key() == account)
1663                    .unwrap_or(false)
1664            })
1665            .count()
1666    }
1667
1668    /// Count connections for a specific metering key
1669    fn count_connections_for_metering_key(&self, metering_key: &str) -> usize {
1670        self.clients
1671            .iter()
1672            .filter(|entry| {
1673                entry
1674                    .value()
1675                    .auth_context
1676                    .as_ref()
1677                    .map(|ctx| ctx.metering_key == metering_key)
1678                    .unwrap_or(false)
1679            })
1680            .count()
1681    }
1682
1683    /// Count connections for a specific origin
1684    fn count_connections_for_origin(&self, origin: &str) -> usize {
1685        self.clients
1686            .iter()
1687            .filter(|entry| {
1688                entry
1689                    .value()
1690                    .auth_context
1691                    .as_ref()
1692                    .and_then(|ctx| ctx.origin.as_ref())
1693                    .map(|o| o == origin)
1694                    .unwrap_or(false)
1695            })
1696            .count()
1697    }
1698
1699    /// Check if a subscription is allowed for the given client.
1700    ///
1701    /// Returns Ok(()) if the subscription is allowed, or an error with a reason if not.
1702    #[allow(clippy::result_large_err)]
1703    pub async fn check_subscription_allowed(&self, client_id: Uuid) -> Result<(), AuthDeny> {
1704        let context = {
1705            let Some(client) = self.clients.get(&client_id) else {
1706                return Ok(());
1707            };
1708            let current_subs = client.subscription_count().await;
1709
1710            // Check max subscriptions per connection (use token limits, fallback to default limits)
1711            if let Some(ref ctx) = client.auth_context {
1712                let max_subs = ctx.limits.max_subscriptions.or_else(|| {
1713                    self.rate_limit_config
1714                        .default_limits
1715                        .as_ref()
1716                        .and_then(|l| l.max_subscriptions)
1717                });
1718                if let Some(max_subs) = max_subs {
1719                    if current_subs >= max_subs as usize {
1720                        return Err(AuthDeny::new(
1721                            crate::websocket::auth::AuthErrorCode::SubscriptionLimitExceeded,
1722                            format!(
1723                                "Subscription limit exceeded: {} of {} subscriptions for client {}",
1724                                current_subs, max_subs, client_id
1725                            ),
1726                        )
1727                        .with_suggested_action(
1728                            "Unsubscribe from an existing view before creating another subscription",
1729                        ));
1730                    }
1731                }
1732            }
1733            client.auth_context.clone()
1734        };
1735
1736        // Signed subscription-create rates, enforced only for v2 tokens that
1737        // carry the corresponding optional limit fields.
1738        if let (Some(rate_limiter), Some(ctx)) = (self.rate_limiter.as_ref(), context.as_ref()) {
1739            if !ctx.is_legacy_policy() {
1740                if let RateLimitResult::Denied { retry_after, limit } = rate_limiter
1741                    .check_subscription_create_for_consumer(
1742                        ctx.consumer_key(),
1743                        ctx.limits.max_subscription_creates_per_minute,
1744                    )
1745                    .await
1746                {
1747                    return Err(AuthDeny::rate_limited(retry_after, "subscription creates")
1748                        .with_context(format!(
1749                        "subscription-create rate limit for consumer {} of {} per minute exceeded",
1750                        ctx.consumer_key(),
1751                        limit
1752                    )));
1753                }
1754
1755                if ctx.account_key.is_some() {
1756                    if let RateLimitResult::Denied { retry_after, limit } = rate_limiter
1757                        .check_subscription_create_for_account(
1758                            ctx.account_key(),
1759                            ctx.account_limits.max_subscription_creates_per_minute,
1760                        )
1761                        .await
1762                    {
1763                        return Err(AuthDeny::rate_limited(
1764                            retry_after,
1765                            "account subscription creates",
1766                        )
1767                        .with_context(format!(
1768                            "subscription-create rate limit for account {} of {} per minute exceeded",
1769                            ctx.account_key(),
1770                            limit
1771                        )));
1772                    }
1773                }
1774            }
1775        }
1776        Ok(())
1777    }
1778
1779    /// Get metering key for a client
1780    pub fn get_metering_key(&self, client_id: Uuid) -> Option<String> {
1781        self.clients.get(&client_id).and_then(|client| {
1782            client
1783                .auth_context
1784                .as_ref()
1785                .map(|ctx| ctx.metering_key.clone())
1786        })
1787    }
1788
1789    /// Get auth context for a client.
1790    pub fn get_auth_context(&self, client_id: Uuid) -> Option<AuthContext> {
1791        self.clients
1792            .get(&client_id)
1793            .and_then(|client| client.auth_context.clone())
1794    }
1795
1796    /// Check if a snapshot request is allowed (based on max_snapshot_rows limit)
1797    ///
1798    /// Uses token limits if available, falls back to default limits from RateLimitConfig.
1799    #[allow(clippy::result_large_err)]
1800    pub fn check_snapshot_allowed(
1801        &self,
1802        client_id: Uuid,
1803        requested_rows: u32,
1804    ) -> Result<(), AuthDeny> {
1805        if let Some(client) = self.clients.get(&client_id) {
1806            if let Some(ref ctx) = client.auth_context {
1807                let max_rows = ctx.limits.max_snapshot_rows.or_else(|| {
1808                    self.rate_limit_config
1809                        .default_limits
1810                        .as_ref()
1811                        .and_then(|l| l.max_snapshot_rows)
1812                });
1813                if let Some(max_rows) = max_rows {
1814                    if requested_rows > max_rows {
1815                        return Err(AuthDeny::new(
1816                            crate::websocket::auth::AuthErrorCode::SnapshotLimitExceeded,
1817                            format!(
1818                                "Snapshot limit exceeded: requested {} rows, max allowed is {} for client {}",
1819                                requested_rows, max_rows, client_id
1820                            ),
1821                        )
1822                        .with_suggested_action(
1823                            "Request fewer rows or lower the snapshotLimit on the subscription",
1824                        ));
1825                    }
1826                }
1827            }
1828        }
1829        Ok(())
1830    }
1831}
1832
1833impl Default for ClientManager {
1834    fn default() -> Self {
1835        Self::new()
1836    }
1837}
1838
1839#[cfg(test)]
1840#[path = "admission_tests.rs"]
1841mod admission_tests;
1842
1843#[cfg(test)]
1844mod tests {
1845    use super::*;
1846    use crate::websocket::auth::AuthContext;
1847    use arete_auth::{KeyClass, Limits};
1848    use std::net::{IpAddr, Ipv4Addr, SocketAddr};
1849
1850    fn create_test_auth_context(subject: &str, limits: Limits) -> AuthContext {
1851        AuthContext {
1852            subject: subject.to_string(),
1853            issuer: "test-issuer".to_string(),
1854            audience: "test-audience".to_string(),
1855            key_class: KeyClass::Publishable,
1856            metering_key: format!("meter-{}", subject),
1857            deployment_id: None,
1858            target_kind: None,
1859            target_id: None,
1860            program_id: None,
1861            program_release_hash: None,
1862            expires_at: u64::MAX,
1863            scope: "read".to_string(),
1864            limits,
1865            plan: None,
1866            origin: None,
1867            client_ip: None,
1868            jti: uuid::Uuid::new_v4().to_string(),
1869            actor_key: None,
1870            account_key: None,
1871            consumer_key: None,
1872            policy_version: None,
1873            account_limits: Limits::default(),
1874            row_scopes: None,
1875        }
1876    }
1877
1878    fn create_v2_auth_context(
1879        consumer: &str,
1880        account: &str,
1881        policy_version: u32,
1882        limits: Limits,
1883        account_limits: Limits,
1884    ) -> AuthContext {
1885        AuthContext {
1886            subject: "user:1".to_string(),
1887            issuer: "test-issuer".to_string(),
1888            audience: "test-audience".to_string(),
1889            key_class: KeyClass::Publishable,
1890            metering_key: account.to_string(),
1891            deployment_id: None,
1892            target_kind: None,
1893            target_id: None,
1894            program_id: None,
1895            program_release_hash: None,
1896            expires_at: u64::MAX,
1897            scope: "read".to_string(),
1898            limits,
1899            plan: Some("pro".to_string()),
1900            origin: None,
1901            client_ip: None,
1902            jti: uuid::Uuid::new_v4().to_string(),
1903            actor_key: Some("user:1".to_string()),
1904            account_key: Some(account.to_string()),
1905            consumer_key: Some(consumer.to_string()),
1906            policy_version: Some(policy_version),
1907            account_limits,
1908            row_scopes: None,
1909        }
1910    }
1911
1912    fn insert_client(manager: &ClientManager, context: AuthContext) -> Uuid {
1913        let (sender, receiver) = mpsc::channel(8);
1914        // Keep the receiver alive so the sender stays open.
1915        std::mem::forget(receiver);
1916        let client_id = Uuid::new_v4();
1917        manager.clients.insert(
1918            client_id,
1919            ClientInfo::new(
1920                client_id,
1921                sender,
1922                Some(context),
1923                create_test_socket_addr("127.0.0.1"),
1924            ),
1925        );
1926        client_id
1927    }
1928
1929    fn create_test_socket_addr(ip: &str) -> SocketAddr {
1930        SocketAddr::new(
1931            ip.parse::<IpAddr>()
1932                .unwrap_or(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))),
1933            12345,
1934        )
1935    }
1936
1937    #[test]
1938    fn test_egress_tracker_basic() {
1939        let mut tracker = EgressTracker::new();
1940
1941        // Should allow bytes within limit
1942        assert!(tracker.record_bytes(500, 1000));
1943        assert_eq!(tracker.current_usage(), 500);
1944
1945        // Should allow more bytes within limit
1946        assert!(tracker.record_bytes(400, 1000));
1947        assert_eq!(tracker.current_usage(), 900);
1948
1949        // Should reject bytes over limit
1950        assert!(!tracker.record_bytes(200, 1000));
1951        assert_eq!(tracker.current_usage(), 900); // Usage shouldn't increase
1952    }
1953
1954    #[test]
1955    fn test_egress_tracker_window_reset() {
1956        let mut tracker = EgressTracker::new();
1957
1958        // Use up the limit
1959        assert!(tracker.record_bytes(100, 100));
1960        assert!(!tracker.record_bytes(1, 100));
1961
1962        // Reset the window
1963        tracker.bytes_this_minute = 0;
1964        tracker.window_start = SystemTime::now() - Duration::from_secs(61);
1965
1966        // Should allow after window reset
1967        assert!(tracker.record_bytes(50, 100));
1968    }
1969
1970    #[test]
1971    fn test_message_rate_tracker_basic() {
1972        let mut tracker = MessageRateTracker::new();
1973
1974        assert!(tracker.record_message(2));
1975        assert_eq!(tracker.current_usage(), 1);
1976
1977        assert!(tracker.record_message(2));
1978        assert_eq!(tracker.current_usage(), 2);
1979
1980        assert!(!tracker.record_message(2));
1981        assert_eq!(tracker.current_usage(), 2);
1982    }
1983
1984    #[tokio::test]
1985    async fn test_client_inbound_message_limit() {
1986        let (tx, _rx) = mpsc::channel(1);
1987        let client = ClientInfo::new(
1988            Uuid::new_v4(),
1989            tx,
1990            Some(create_test_auth_context(
1991                "user-1",
1992                Limits {
1993                    max_messages_per_minute: Some(2),
1994                    ..Default::default()
1995                },
1996            )),
1997            create_test_socket_addr("127.0.0.1"),
1998        );
1999
2000        assert_eq!(client.record_inbound_message(), Some(1));
2001        assert_eq!(client.record_inbound_message(), Some(2));
2002        assert_eq!(client.record_inbound_message(), None);
2003    }
2004
2005    #[tokio::test]
2006    async fn duplicate_subscription_id_is_rejected_without_replacement() {
2007        let (tx, _rx) = mpsc::channel(1);
2008        let client = ClientInfo::new(
2009            Uuid::new_v4(),
2010            tx,
2011            None,
2012            create_test_socket_addr("127.0.0.1"),
2013        );
2014        let first = CancellationToken::new();
2015        let duplicate = CancellationToken::new();
2016
2017        assert!(
2018            client
2019                .add_subscription("opaque-id".to_string(), first.clone())
2020                .await
2021        );
2022        assert!(
2023            !client
2024                .add_subscription("opaque-id".to_string(), duplicate.clone())
2025                .await
2026        );
2027        assert!(!first.is_cancelled());
2028        assert!(!duplicate.is_cancelled());
2029
2030        assert!(client.remove_subscription("opaque-id").await);
2031        assert!(first.is_cancelled());
2032        assert!(!duplicate.is_cancelled());
2033    }
2034
2035    #[tokio::test]
2036    async fn test_no_limits() {
2037        let manager = ClientManager::new();
2038        let addr = create_test_socket_addr("127.0.0.1");
2039
2040        // No auth context - should succeed
2041        assert!(manager.check_connection_allowed(addr, &None).await.is_ok());
2042
2043        // Auth context with no limits - should succeed
2044        let auth_context = create_test_auth_context("test", Limits::default());
2045        assert!(manager
2046            .check_connection_allowed(addr, &Some(auth_context))
2047            .await
2048            .is_ok());
2049    }
2050
2051    #[tokio::test]
2052    async fn test_per_subject_connection_limit() {
2053        let manager = ClientManager::new();
2054
2055        let limits = Limits {
2056            max_connections: Some(2),
2057            ..Default::default()
2058        };
2059
2060        let auth_context = create_test_auth_context("user-1", limits);
2061        let addr = create_test_socket_addr("127.0.0.1");
2062
2063        // First connection should succeed (no clients yet)
2064        assert!(manager
2065            .check_connection_allowed(addr, &Some(auth_context.clone()))
2066            .await
2067            .is_ok());
2068    }
2069
2070    #[tokio::test]
2071    async fn test_per_ip_connection_limit() {
2072        let manager = ClientManager::new().with_max_connections_per_ip(2);
2073        let addr = create_test_socket_addr("192.168.1.1");
2074
2075        // Should succeed when no connections from that IP
2076        assert!(manager.check_connection_allowed(addr, &None).await.is_ok());
2077    }
2078
2079    // Tests for RateLimitConfig
2080    #[test]
2081    fn rate_limit_config_default() {
2082        let config = RateLimitConfig::default();
2083        assert!(config.max_connections_per_ip.is_none());
2084        assert_eq!(config.client_timeout, Duration::from_secs(300));
2085        assert_eq!(config.message_queue_size, 512);
2086        assert!(config.max_reconnect_attempts.is_none());
2087        assert_eq!(config.message_rate_window, Duration::from_secs(60));
2088        assert_eq!(config.egress_rate_window, Duration::from_secs(60));
2089    }
2090
2091    #[test]
2092    fn rate_limit_config_builder_methods() {
2093        let config = RateLimitConfig::default()
2094            .with_max_connections_per_ip(10)
2095            .with_timeout(Duration::from_secs(600))
2096            .with_message_queue_size(1024)
2097            .with_rate_limit_window(Duration::from_secs(120));
2098
2099        assert_eq!(config.max_connections_per_ip, Some(10));
2100        assert_eq!(config.client_timeout, Duration::from_secs(600));
2101        assert_eq!(config.message_queue_size, 1024);
2102        assert_eq!(config.message_rate_window, Duration::from_secs(120));
2103        assert_eq!(config.egress_rate_window, Duration::from_secs(120));
2104    }
2105
2106    #[tokio::test]
2107    async fn client_manager_with_config() {
2108        let config = RateLimitConfig::default()
2109            .with_max_connections_per_ip(5)
2110            .with_timeout(Duration::from_secs(120))
2111            .with_message_queue_size(256);
2112
2113        let manager = ClientManager::with_config(config);
2114        let addr = create_test_socket_addr("10.0.0.1");
2115
2116        // Check that the configuration was applied
2117        assert_eq!(manager.rate_limit_config().max_connections_per_ip, Some(5));
2118        assert_eq!(
2119            manager.rate_limit_config().client_timeout,
2120            Duration::from_secs(120)
2121        );
2122        assert_eq!(manager.rate_limit_config().message_queue_size, 256);
2123
2124        // Should allow when under limit
2125        assert!(manager.check_connection_allowed(addr, &None).await.is_ok());
2126    }
2127
2128    #[tokio::test]
2129    async fn client_manager_builder_pattern() {
2130        let manager = ClientManager::new()
2131            .with_max_connections_per_ip(10)
2132            .with_timeout(Duration::from_secs(180))
2133            .with_message_queue_size(1024)
2134            .with_rate_limit_window(Duration::from_secs(90));
2135
2136        assert_eq!(manager.rate_limit_config().max_connections_per_ip, Some(10));
2137        assert_eq!(
2138            manager.rate_limit_config().client_timeout,
2139            Duration::from_secs(180)
2140        );
2141        assert_eq!(manager.rate_limit_config().message_queue_size, 1024);
2142        assert_eq!(
2143            manager.rate_limit_config().message_rate_window,
2144            Duration::from_secs(90)
2145        );
2146    }
2147
2148    // Integration test: Connection limits are enforced
2149    #[tokio::test]
2150    async fn connection_limit_enforcement_with_actual_clients() {
2151        let manager = ClientManager::new().with_max_connections_per_ip(2);
2152        let addr1 = create_test_socket_addr("192.168.1.1");
2153        let addr2 = create_test_socket_addr("192.168.1.2");
2154
2155        // First connection from IP1 should succeed
2156        let auth1 = create_test_auth_context("user-1", Limits::default());
2157        assert!(manager
2158            .check_connection_allowed(addr1, &Some(auth1.clone()))
2159            .await
2160            .is_ok());
2161
2162        // Simulate adding a client (we can't easily do this without a real WebSocket,
2163        // but we can verify the check logic works)
2164
2165        // Same IP, different auth context - should still count toward IP limit
2166        let auth2 = create_test_auth_context("user-2", Limits::default());
2167        assert!(manager
2168            .check_connection_allowed(addr1, &Some(auth2.clone()))
2169            .await
2170            .is_ok());
2171
2172        // Different IP - should succeed regardless
2173        let auth3 = create_test_auth_context("user-3", Limits::default());
2174        assert!(manager
2175            .check_connection_allowed(addr2, &Some(auth3.clone()))
2176            .await
2177            .is_ok());
2178    }
2179
2180    // Test subscription limit enforcement
2181    #[tokio::test]
2182    async fn subscription_limit_enforcement() {
2183        let manager = ClientManager::new();
2184        let addr = create_test_socket_addr("127.0.0.1");
2185
2186        // Create auth context with subscription limit
2187        let auth = create_test_auth_context(
2188            "user-1",
2189            Limits {
2190                max_subscriptions: Some(2),
2191                ..Default::default()
2192            },
2193        );
2194
2195        // Check should pass initially
2196        assert!(manager
2197            .check_connection_allowed(addr, &Some(auth.clone()))
2198            .await
2199            .is_ok());
2200
2201        // Note: We can't easily test the full subscription flow without a real connection,
2202        // but we verify the limit configuration is properly stored
2203        assert_eq!(auth.limits.max_subscriptions, Some(2));
2204    }
2205
2206    // Test snapshot limit enforcement
2207    #[tokio::test]
2208    async fn snapshot_limit_enforcement() {
2209        let manager = ClientManager::new();
2210        let addr = create_test_socket_addr("127.0.0.1");
2211
2212        let auth = create_test_auth_context(
2213            "user-1",
2214            Limits {
2215                max_snapshot_rows: Some(1000),
2216                ..Default::default()
2217            },
2218        );
2219
2220        assert!(manager
2221            .check_connection_allowed(addr, &Some(auth.clone()))
2222            .await
2223            .is_ok());
2224
2225        // Note: Actual snapshot limit checking happens in check_snapshot_allowed
2226        // which requires a connected client
2227    }
2228
2229    #[tokio::test]
2230    async fn two_consumers_share_the_account_connection_cap() {
2231        let manager = ClientManager::new();
2232        let account_limits = Limits {
2233            max_connections: Some(1),
2234            ..Limits::default()
2235        };
2236        let consumer_a = create_v2_auth_context(
2237            "consumer:a",
2238            "account:42",
2239            1,
2240            Limits::default(),
2241            account_limits.clone(),
2242        );
2243        insert_client(&manager, consumer_a);
2244
2245        // A different consumer on the same account is blocked by the
2246        // aggregate cap.
2247        let consumer_b = create_v2_auth_context(
2248            "consumer:b",
2249            "account:42",
2250            1,
2251            Limits::default(),
2252            account_limits.clone(),
2253        );
2254        let addr = create_test_socket_addr("127.0.0.1");
2255        assert!(manager
2256            .check_connection_allowed(addr, &Some(consumer_b))
2257            .await
2258            .is_err());
2259
2260        // A consumer on another account is unaffected.
2261        let other_account = create_v2_auth_context(
2262            "consumer:c",
2263            "account:43",
2264            1,
2265            Limits::default(),
2266            account_limits,
2267        );
2268        assert!(manager
2269            .check_connection_allowed(addr, &Some(other_account))
2270            .await
2271            .is_ok());
2272    }
2273
2274    #[tokio::test]
2275    async fn publishable_sessions_see_a_fixed_connection_limit_reason() {
2276        let manager = ClientManager::new();
2277        let account_limits = Limits {
2278            max_connections: Some(1),
2279            ..Limits::default()
2280        };
2281        insert_client(
2282            &manager,
2283            create_v2_auth_context(
2284                "consumer:a",
2285                "account:42",
2286                1,
2287                Limits::default(),
2288                account_limits.clone(),
2289            ),
2290        );
2291        let addr = create_test_socket_addr("127.0.0.1");
2292
2293        let visitor = create_v2_auth_context(
2294            "consumer:b",
2295            "account:42",
2296            1,
2297            Limits::default(),
2298            account_limits.clone(),
2299        );
2300        let deny = manager
2301            .check_connection_allowed(addr, &Some(visitor))
2302            .await
2303            .unwrap_err();
2304        assert_eq!(deny.code, AuthErrorCode::ConnectionLimitExceeded);
2305        let response = serde_json::to_string(&deny.to_error_response()).unwrap();
2306        assert_eq!(deny.reason, "Connection limit reached");
2307        for leaked in ["account:42", "consumer:b", "1 of 1"] {
2308            assert!(!response.contains(leaked), "{leaked} in {response}");
2309        }
2310
2311        // A secret-key session keeps the detailed reason.
2312        let mut server = create_v2_auth_context(
2313            "consumer:c",
2314            "account:42",
2315            1,
2316            Limits::default(),
2317            account_limits,
2318        );
2319        server.key_class = KeyClass::Secret;
2320        let deny = manager
2321            .check_connection_allowed(addr, &Some(server))
2322            .await
2323            .unwrap_err();
2324        assert!(
2325            deny.reason.contains("account account:42"),
2326            "{}",
2327            deny.reason
2328        );
2329    }
2330
2331    #[tokio::test]
2332    async fn one_consumer_is_limited_independently_of_its_account() {
2333        let manager = ClientManager::new();
2334        let limits = Limits {
2335            max_connections: Some(1),
2336            ..Limits::default()
2337        };
2338        let account_limits = Limits {
2339            max_connections: Some(10),
2340            ..Limits::default()
2341        };
2342        let consumer_a = create_v2_auth_context(
2343            "consumer:a",
2344            "account:42",
2345            1,
2346            limits.clone(),
2347            account_limits.clone(),
2348        );
2349        insert_client(&manager, consumer_a.clone());
2350
2351        let addr = create_test_socket_addr("127.0.0.1");
2352        // The same consumer hits its own cap.
2353        assert!(manager
2354            .check_connection_allowed(addr, &Some(consumer_a))
2355            .await
2356            .is_err());
2357
2358        // A sibling consumer under the same account is still admitted.
2359        let consumer_b =
2360            create_v2_auth_context("consumer:b", "account:42", 1, limits, account_limits);
2361        assert!(manager
2362            .check_connection_allowed(addr, &Some(consumer_b))
2363            .await
2364            .is_ok());
2365    }
2366
2367    #[tokio::test]
2368    async fn policy_version_upgrade_stale_and_conflict_rules_apply() {
2369        let manager = ClientManager::new();
2370        let addr = create_test_socket_addr("127.0.0.1");
2371        let limits_v1 = Limits {
2372            max_connections: Some(5),
2373            ..Limits::default()
2374        };
2375        let limits_v2 = Limits {
2376            max_connections: Some(2),
2377            ..Limits::default()
2378        };
2379
2380        // Version 1 admits and creates state.
2381        let v1 = create_v2_auth_context(
2382            "consumer:a",
2383            "account:42",
2384            1,
2385            Limits::default(),
2386            limits_v1.clone(),
2387        );
2388        assert!(manager
2389            .check_connection_allowed(addr, &Some(v1.clone()))
2390            .await
2391            .is_ok());
2392
2393        // Version 2 with new limits replaces the policy.
2394        let v2 = create_v2_auth_context(
2395            "consumer:a",
2396            "account:42",
2397            2,
2398            Limits::default(),
2399            limits_v2.clone(),
2400        );
2401        assert!(manager
2402            .check_connection_allowed(addr, &Some(v2))
2403            .await
2404            .is_ok());
2405
2406        // A stale version-1 token is rejected once version 2 was observed.
2407        let stale = manager.check_connection_allowed(addr, &Some(v1)).await;
2408        assert!(stale.is_err());
2409
2410        // Version 2 with different limits is a signing/config fault.
2411        let conflict =
2412            create_v2_auth_context("consumer:a", "account:42", 2, Limits::default(), limits_v1);
2413        assert!(manager
2414            .check_connection_allowed(addr, &Some(conflict))
2415            .await
2416            .is_err());
2417    }
2418
2419    #[tokio::test]
2420    async fn legacy_tokens_keep_old_behavior_and_are_counted() {
2421        let manager = ClientManager::new();
2422        let addr = create_test_socket_addr("127.0.0.1");
2423        let legacy = create_test_auth_context(
2424            "user-1",
2425            Limits {
2426                max_connections: Some(2),
2427                ..Limits::default()
2428            },
2429        );
2430        assert!(legacy.is_legacy_policy());
2431
2432        assert!(manager
2433            .check_connection_allowed(addr, &Some(legacy.clone()))
2434            .await
2435            .is_ok());
2436        assert_eq!(manager.account_policies().legacy_token_count(), 1);
2437        // Legacy tokens create no account policy state.
2438        assert_eq!(manager.account_policies().tracked_accounts(), 0);
2439
2440        // Legacy connection counting still keys off the subject.
2441        insert_client(&manager, legacy.clone());
2442        insert_client(&manager, legacy.clone());
2443        assert!(manager
2444            .check_connection_allowed(addr, &Some(legacy))
2445            .await
2446            .is_err());
2447    }
2448
2449    #[tokio::test]
2450    async fn account_message_budget_is_shared_across_consumers() {
2451        let manager = ClientManager::new();
2452        let account_limits = Limits {
2453            max_messages_per_minute: Some(2),
2454            ..Limits::default()
2455        };
2456        let client_a = insert_client(
2457            &manager,
2458            create_v2_auth_context(
2459                "consumer:a",
2460                "account:42",
2461                1,
2462                Limits::default(),
2463                account_limits.clone(),
2464            ),
2465        );
2466        let client_b = insert_client(
2467            &manager,
2468            create_v2_auth_context(
2469                "consumer:b",
2470                "account:42",
2471                1,
2472                Limits::default(),
2473                account_limits,
2474            ),
2475        );
2476
2477        assert!(manager.check_inbound_message_allowed(client_a).is_ok());
2478        assert!(manager.check_inbound_message_allowed(client_b).is_ok());
2479        // The third message anywhere on the account is rejected.
2480        assert!(manager.check_inbound_message_allowed(client_a).is_err());
2481    }
2482
2483    #[tokio::test]
2484    async fn account_state_evicts_when_idle_and_unreferenced() {
2485        let manager = ClientManager::new();
2486        let addr = create_test_socket_addr("127.0.0.1");
2487        let context = create_v2_auth_context(
2488            "consumer:a",
2489            "account:42",
2490            1,
2491            Limits::default(),
2492            Limits::default(),
2493        );
2494        assert!(manager
2495            .check_connection_allowed(addr, &Some(context))
2496            .await
2497            .is_ok());
2498        assert_eq!(manager.account_policies().tracked_accounts(), 1);
2499
2500        // No live connection references the account; an idle sweep with a
2501        // zero TTL registry drops it.
2502        manager.account_policies().evict_idle(|_| false);
2503        assert_eq!(
2504            manager.account_policies().tracked_accounts(),
2505            1,
2506            "TTL keeps fresh entries"
2507        );
2508        manager.cleanup_account_state();
2509        assert_eq!(
2510            manager.account_policies().tracked_accounts(),
2511            1,
2512            "fresh entries survive sweep"
2513        );
2514    }
2515
2516    #[test]
2517    fn expired_token_is_removed_without_deadlocking() {
2518        let manager = ClientManager::new();
2519        let mut context = create_test_auth_context("user-1", Limits::default());
2520        context.expires_at = 1;
2521        let client_id = insert_client(&manager, context);
2522
2523        // Send from another thread: a deadlock blocks that thread for good,
2524        // so the test fails on the timeout instead of hanging.
2525        let (done_tx, done_rx) = std::sync::mpsc::channel();
2526        let sender = manager.clone();
2527        std::thread::spawn(move || {
2528            let result = sender.send_to_client(client_id, Arc::new(Bytes::from_static(b"update")));
2529            let _ = done_tx.send(result);
2530        });
2531
2532        let result = done_rx
2533            .recv_timeout(Duration::from_secs(5))
2534            .expect("send_to_client deadlocked on an expired token");
2535        assert_eq!(result, Err(SendError::ClientDisconnected));
2536        assert!(!manager.has_client(client_id));
2537    }
2538
2539    #[tokio::test]
2540    async fn an_expired_client_keeps_its_close_reason_when_its_queue_is_full() {
2541        let manager = ClientManager::new();
2542        let (sender, mut queue) = mpsc::channel(1);
2543        sender
2544            .try_send(Message::Text("backlog".into()))
2545            .expect("room for one message");
2546        let client_id = Uuid::new_v4();
2547        let mut context = create_test_auth_context("user-1", Limits::default());
2548        context.expires_at = 1;
2549        let client = ClientInfo::new(
2550            client_id,
2551            sender,
2552            Some(context),
2553            create_test_socket_addr("127.0.0.1"),
2554        );
2555        let close_frame = client.close_frame.clone();
2556        manager.clients.insert(client_id, client);
2557
2558        assert!(manager.check_and_remove_expired(client_id));
2559
2560        let frame = close_frame
2561            .get()
2562            .expect("the close reason is recorded even though the queue is full");
2563        assert_eq!(frame.code, CloseCode::Policy);
2564        assert_eq!(
2565            frame.reason.as_str(),
2566            "token-expired: Authentication token expired"
2567        );
2568        // The backlog is still delivered first, and removing the client
2569        // dropped its sender, so the queue then ends and the sender task
2570        // closes the socket with the recorded reason.
2571        assert!(matches!(queue.recv().await, Some(Message::Text(_))));
2572        assert!(queue.recv().await.is_none());
2573    }
2574
2575    #[test]
2576    fn unexpired_token_is_kept() {
2577        let manager = ClientManager::new();
2578        let client_id = insert_client(
2579            &manager,
2580            create_test_auth_context("user-1", Limits::default()),
2581        );
2582
2583        assert!(!manager.check_and_remove_expired(client_id));
2584        assert!(manager.has_client(client_id));
2585    }
2586
2587    // Test WebSocketRateLimiter integration
2588    #[tokio::test]
2589    async fn test_rate_limiter_integration() {
2590        use crate::websocket::rate_limiter::{RateLimiterConfig, WebSocketRateLimiter};
2591
2592        let rate_limiter = Arc::new(WebSocketRateLimiter::new(RateLimiterConfig::default()));
2593        let manager = ClientManager::new().with_rate_limiter(rate_limiter);
2594        let addr = create_test_socket_addr("127.0.0.1");
2595
2596        // Should allow connections when rate limiter is configured
2597        let auth = create_test_auth_context("user-1", Limits::default());
2598        assert!(manager
2599            .check_connection_allowed(addr, &Some(auth))
2600            .await
2601            .is_ok());
2602    }
2603}