1use std::{
7 collections::BTreeMap,
8 fmt,
9 sync::{
10 Arc, Weak,
11 atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
12 },
13 time::Duration,
14};
15
16use data_encoding::BASE64;
17use futures_util::{SinkExt as _, StreamExt as _};
18use parking_lot::Mutex as ParkingMutex;
19use rand::{TryRng as _, rngs::SysRng};
20use reqwest::{StatusCode, Version, header};
21use serde::{Deserialize, Serialize, de::DeserializeOwned};
22use serde_json::Value;
23use serde_json::value::RawValue;
24use sha1::{Digest as _, Sha1};
25use thiserror::Error;
26use tokio::{
27 sync::{Notify, mpsc, oneshot},
28 task::JoinHandle,
29 time::Instant,
30};
31use tokio_tungstenite::{
32 WebSocketStream,
33 tungstenite::{
34 Error as TungsteniteError, Message,
35 protocol::{Role, WebSocketConfig},
36 },
37};
38use tokio_util::sync::CancellationToken;
39
40use crate::{AccountId, ContractId, Endpoints, Error as ClientError, token::TokenStore};
41
42const SIGNALR_TERMINATOR: char = '\u{001e}';
43const SIGNALR_PING: &str = "{\"type\":6}\u{001e}";
44const WEBSOCKET_READ_BUFFER_SIZE: usize = 64 * 1_024;
46const WEBSOCKET_WRITE_BUFFER_SIZE: usize = 64 * 1_024;
47const EVENT_BASE_WEIGHT: usize = 256;
48const COALESCED_RECORD_QUANTUM: usize = 32;
49const CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
50const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10);
51const CLOSE_TIMEOUT: Duration = Duration::from_secs(5);
52const CLIENT_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(15);
53const SOCKET_PROBE_INTERVAL: Duration = Duration::from_secs(15);
56const WATCHDOG_INTERVAL: Duration = Duration::from_secs(5);
57const WEBSOCKET_GUID: &[u8] = b"258EAFA5-E914-47DA-95CA-C5AB0DC85B11";
58
59type PendingInvocation = oneshot::Sender<Result<(), ()>>;
60
61struct PendingEntry {
62 generation: u64,
63 reply: PendingInvocation,
64}
65
66type PendingInvocations = Arc<ParkingMutex<BTreeMap<String, PendingEntry>>>;
67
68#[derive(Serialize)]
69#[serde(rename_all = "camelCase")]
70struct OutboundInvocation<'a> {
71 #[serde(rename = "type")]
72 message_type: u8,
73 invocation_id: &'a str,
74 target: &'a str,
75 arguments: &'a [Value],
76}
77
78#[derive(Clone, Copy, Debug, Eq, PartialEq)]
80#[non_exhaustive]
81pub enum Hub {
82 Market,
84 User,
86}
87
88impl Hub {
89 const fn path(self) -> &'static str {
90 match self {
91 Self::Market => "market",
92 Self::User => "user",
93 }
94 }
95}
96
97#[derive(Clone, Debug)]
99pub struct SignalRInvocation {
100 target: String,
101 contract_id: Option<ContractId>,
102 payload: Value,
103 raw_entity: Box<RawValue>,
104}
105
106impl PartialEq for SignalRInvocation {
107 fn eq(&self, other: &Self) -> bool {
108 self.target == other.target
109 && self.contract_id == other.contract_id
110 && self.payload == other.payload
111 && self.raw_entity.get() == other.raw_entity.get()
112 }
113}
114
115impl SignalRInvocation {
116 pub fn from_value(value: &Value) -> Result<Option<Self>, RealtimeError> {
131 let discriminator = value
132 .get("type")
133 .ok_or(RealtimeError::Protocol("message type is missing"))?;
134 match discriminator.as_u64() {
135 None => {
136 return Err(RealtimeError::Protocol(
137 "message type was not an unsigned integer",
138 ));
139 }
140 Some(1) => {}
141 Some(_) => return Ok(None),
142 }
143 let object = value.as_object().ok_or(RealtimeError::Protocol(
144 "invocation frame was not an object",
145 ))?;
146 let target = object
147 .get("target")
148 .and_then(|target| target.as_str())
149 .ok_or(RealtimeError::Protocol("invocation target is missing"))?;
150 let arguments = object
151 .get("arguments")
152 .ok_or(RealtimeError::Protocol("invocation arguments are missing"))?
153 .as_array()
154 .ok_or(RealtimeError::Protocol(
155 "invocation arguments were not an array",
156 ))?;
157 let mut arguments = arguments.iter();
158 let first = arguments
159 .next()
160 .ok_or(RealtimeError::Protocol("invocation payload is missing"))?;
161 let (contract_id, payload) = match arguments.next() {
162 Some(second) => {
163 let contract_id = ContractId::new(first.as_str().ok_or(
164 RealtimeError::Protocol("market contract identifier was not a string"),
165 )?)
166 .map_err(|_| RealtimeError::Protocol("contract identifier is invalid"))?;
167 (Some(contract_id), second)
168 }
169 None => (None, first),
170 };
171 if arguments.next().is_some() {
172 return Err(RealtimeError::Protocol(
173 "invocation contained too many arguments",
174 ));
175 }
176 let entity = payload
177 .get("data")
178 .filter(|data| data.is_object() || data.is_array())
179 .unwrap_or(payload);
180 let raw_entity =
181 RawValue::from_string(serde_json::to_string(entity).map_err(RealtimeError::Decode)?)
182 .map_err(RealtimeError::Decode)?;
183 Ok(Some(Self {
184 target: target.to_owned(),
185 contract_id,
186 payload: payload.clone(),
187 raw_entity,
188 }))
189 }
190
191 pub fn from_json(json: &str) -> Result<Option<Self>, RealtimeError> {
202 #[derive(Deserialize)]
203 struct InvocationFrame {
204 #[serde(rename = "type")]
205 message_type: u64,
206 target: Option<String>,
207 arguments: Option<Vec<Box<RawValue>>>,
208 }
209
210 let frame: InvocationFrame = serde_json::from_str(json).map_err(RealtimeError::Decode)?;
211 if frame.message_type != 1 {
212 return Ok(None);
213 }
214 let target = frame
215 .target
216 .ok_or(RealtimeError::Protocol("invocation target is missing"))?;
217 let mut arguments = frame
218 .arguments
219 .ok_or(RealtimeError::Protocol("invocation arguments are missing"))?
220 .into_iter();
221 let first = arguments
222 .next()
223 .ok_or(RealtimeError::Protocol("invocation payload is missing"))?;
224 let (contract_id, raw_payload) = match arguments.next() {
225 Some(second) => {
226 let contract = serde_json::from_str::<String>(first.get()).map_err(|_| {
227 RealtimeError::Protocol("market contract identifier was not a string")
228 })?;
229 let contract_id = ContractId::new(contract)
230 .map_err(|_| RealtimeError::Protocol("contract identifier is invalid"))?;
231 (Some(contract_id), second)
232 }
233 None => (None, first),
234 };
235 if arguments.next().is_some() {
236 return Err(RealtimeError::Protocol(
237 "invocation contained too many arguments",
238 ));
239 }
240 let payload: Value =
241 serde_json::from_str(raw_payload.get()).map_err(RealtimeError::Decode)?;
242 let raw_entity = if payload
243 .get("data")
244 .is_some_and(|data| data.is_object() || data.is_array())
245 {
246 let mut object =
247 serde_json::from_str::<BTreeMap<String, Box<RawValue>>>(raw_payload.get())
248 .map_err(RealtimeError::Decode)?;
249 object.remove("data").ok_or(RealtimeError::Protocol(
250 "invocation data envelope is missing",
251 ))?
252 } else {
253 raw_payload
254 };
255 Ok(Some(Self {
256 target,
257 contract_id,
258 payload,
259 raw_entity,
260 }))
261 }
262
263 #[must_use]
265 pub fn target(&self) -> &str {
266 &self.target
267 }
268
269 #[must_use]
271 pub fn contract_id(&self) -> Option<&ContractId> {
272 self.contract_id.as_ref()
273 }
274
275 #[must_use]
277 pub fn payload(&self) -> &Value {
278 &self.payload
279 }
280
281 #[must_use]
284 pub fn entity(&self) -> &Value {
285 self.payload
286 .get("data")
287 .filter(|data| data.is_object() || data.is_array())
288 .unwrap_or(&self.payload)
289 }
290
291 pub fn decode<T>(&self) -> Result<T, RealtimeError>
297 where
298 T: DeserializeOwned,
299 {
300 serde_json::from_str(self.raw_entity.get()).map_err(RealtimeError::Decode)
301 }
302
303 #[must_use]
309 pub fn decode_batch<T>(&self) -> Vec<Result<T, RealtimeError>>
310 where
311 T: DeserializeOwned,
312 {
313 if !self.entity().is_array() {
314 return vec![self.decode()];
315 }
316 let values = match serde_json::from_str::<Vec<Box<RawValue>>>(self.raw_entity.get()) {
317 Ok(values) => values,
318 Err(error) => return vec![Err(RealtimeError::Decode(error))],
319 };
320 values
321 .into_iter()
322 .filter(|value| value.get() != "null")
323 .map(|value| serde_json::from_str(value.get()).map_err(RealtimeError::Decode))
324 .collect()
325 }
326}
327
328#[derive(Clone, Debug, PartialEq)]
330#[non_exhaustive]
331pub enum RealtimeEvent {
332 Connected,
334 Disconnected,
336 Reconnected,
341 TransportGap,
346 Invocation(SignalRInvocation),
348 Message(Value),
353}
354
355mod config;
356mod event_flow;
357pub(crate) use config::RealtimeConfig;
358mod session_handle;
359use event_flow::{EventEnvelope, EventFlow, PublishOutcome};
360pub use event_flow::{RealtimeEventReceiver, RealtimeGeneration, RealtimeMessage};
361pub use session_handle::RealtimeSession;
362
363#[derive(Debug, Error)]
365#[non_exhaustive]
366pub enum RealtimeError {
367 #[error("authentication is required before connecting a real-time hub")]
369 MissingAuthToken,
370 #[error("real-time endpoint configuration is invalid")]
372 Endpoint(#[source] ClientError),
373 #[error("WebSocket transport failed")]
376 Transport,
377 #[error("WebSocket ping/pong check timed out")]
379 PingTimedOut,
380 #[error("SignalR handshake failed: {0}")]
382 Handshake(&'static str),
383 #[error("SignalR protocol error: {0}")]
385 Protocol(&'static str),
386 #[error("SignalR JSON decoding failed")]
388 Decode(#[source] serde_json::Error),
389 #[error("SignalR invocation JSON encoding failed")]
391 Encode(#[source] serde_json::Error),
392 #[error("real-time client is already connected")]
394 AlreadyConnected,
395 #[error("real-time connection setup was cancelled")]
397 ConnectionCancelled,
398 #[error("real-time WebSocket upgrade timed out")]
400 ConnectionTimedOut,
401 #[error("real-time client is not connected")]
403 NotConnected,
404 #[error("real-time socket generation is no longer current")]
406 StaleGeneration,
407 #[error("subscription is not valid for this real-time hub")]
409 WrongHub,
410 #[error("real-time writer queue is full")]
412 SendQueueFull,
413 #[error("real-time writer is closed")]
415 SendClosed,
416 #[error("real-time reconnect is fenced by an unacknowledged transport gap")]
418 TransportGapPending,
419 #[error("real-time event receiver is closed")]
421 EventReceiverClosed,
422 #[error("pending SignalR invocation capacity is exhausted")]
424 PendingInvocationCapacity,
425 #[error("real-time transport identifier capacity is exhausted")]
427 IdentifierCapacity,
428 #[error("SignalR invocation `{target}` was rejected")]
430 InvocationRejected {
431 target: String,
433 },
434 #[error("SignalR invocation `{target}` timed out")]
436 InvocationTimedOut {
437 target: String,
439 },
440 #[error("SignalR invocation `{target}` ended with its session")]
442 InvocationSessionEnded {
443 target: String,
445 },
446 #[error("real-time close handshake did not complete")]
448 Close,
449}
450
451pub struct RealtimeClient {
457 inner: Arc<RealtimeInner>,
458}
459
460struct RealtimeInner {
461 hub: Hub,
462 endpoints: Endpoints,
463 http: reqwest::Client,
464 token: Arc<TokenStore>,
465 config: RealtimeConfig,
466 lifecycle: ParkingMutex<Lifecycle>,
467 lifecycle_changed: Notify,
468 generation: AtomicU64,
469 reconnect_enabled: AtomicBool,
470 owner_cancel: CancellationToken,
471 client_handles: AtomicUsize,
472 watchdog_task: ParkingMutex<Option<JoinHandle<()>>>,
473 retirement_task: ParkingMutex<Option<JoinHandle<()>>>,
474 failed_close: AtomicU64,
475 last_activity: ParkingMutex<Instant>,
476 socket_probe: ParkingMutex<Option<(u64, u64)>>,
477 request_counter: AtomicU64,
478 pending: PendingInvocations,
479 event_tx: mpsc::UnboundedSender<EventEnvelope>,
480 event_rx: ParkingMutex<Option<RealtimeEventReceiver>>,
481 event_flow: Arc<EventFlow>,
482}
483
484enum Lifecycle {
485 Disconnected,
486 Connecting {
487 generation: u64,
488 cancellation: CancellationToken,
489 },
490 Connected(Session),
491 Closing {
492 generation: u64,
493 was_connected: bool,
494 },
495}
496
497impl Lifecycle {
498 fn generation(&self) -> Option<u64> {
499 match self {
500 Self::Disconnected => None,
501 Self::Connecting { generation, .. } | Self::Closing { generation, .. } => {
502 Some(*generation)
503 }
504 Self::Connected(session) => Some(session.generation),
505 }
506 }
507}
508
509struct Session {
510 runtime: tokio::runtime::Handle,
511 generation: u64,
512 writer: mpsc::Sender<Message>,
513 cancellation: CancellationToken,
514 reader_task: JoinHandle<Result<(), RealtimeError>>,
515 writer_task: JoinHandle<Result<(), RealtimeError>>,
516}
517
518struct ConnectClaim {
519 inner: Weak<RealtimeInner>,
520 generation: u64,
521 cancellation: CancellationToken,
522 complete: bool,
523}
524
525impl ConnectClaim {
526 fn complete(&mut self) {
527 self.complete = true;
528 }
529}
530
531impl Drop for ConnectClaim {
532 fn drop(&mut self) {
533 if !self.complete
534 && let Some(inner) = self.inner.upgrade()
535 {
536 inner.cancel_connect(self.generation);
537 }
538 }
539}
540
541struct PendingGuard {
542 pending: PendingInvocations,
543 invocation_id: String,
544 generation: u64,
545}
546
547impl Drop for PendingGuard {
548 fn drop(&mut self) {
549 {
550 let mut pending = self.pending.lock();
551 if pending
552 .get(&self.invocation_id)
553 .is_some_and(|entry| entry.generation == self.generation)
554 {
555 pending.remove(&self.invocation_id);
556 }
557 }
558 }
559}
560
561enum DisconnectAction {
562 None,
563 Wait(u64),
564 Close(Session),
565}
566
567#[derive(Clone, Copy, Debug, Eq, PartialEq)]
568enum ProcessOutcome {
569 Continue,
570 Close,
571}
572
573impl Drop for Session {
574 fn drop(&mut self) {
575 self.cancellation.cancel();
576 self.reader_task.abort();
577 self.writer_task.abort();
578 }
579}
580
581impl RealtimeClient {
582 pub(crate) fn new(
583 hub: Hub,
584 endpoints: Endpoints,
585 http: reqwest::Client,
586 token: Arc<TokenStore>,
587 config: RealtimeConfig,
588 ) -> Self {
589 let (event_tx, event_rx) = mpsc::unbounded_channel();
590 let event_flow = EventFlow::new();
591 Self {
592 inner: Arc::new(RealtimeInner {
593 hub,
594 endpoints,
595 http,
596 token,
597 config,
598 lifecycle: ParkingMutex::new(Lifecycle::Disconnected),
599 lifecycle_changed: Notify::new(),
600 generation: AtomicU64::new(0),
601 reconnect_enabled: AtomicBool::new(false),
602 owner_cancel: CancellationToken::new(),
603 client_handles: AtomicUsize::new(1),
604 watchdog_task: ParkingMutex::new(None),
605 retirement_task: ParkingMutex::new(None),
606 failed_close: AtomicU64::new(0),
607 last_activity: ParkingMutex::new(Instant::now()),
608 socket_probe: ParkingMutex::new(None),
609 request_counter: AtomicU64::new(1),
610 pending: Arc::default(),
611 event_tx,
612 event_rx: ParkingMutex::new(Some(RealtimeEventReceiver {
613 events: event_rx,
614 flow: Arc::clone(&event_flow),
615 gap_reported: false,
616 })),
617 event_flow,
618 }),
619 }
620 }
621
622 #[must_use]
624 pub fn take_event_receiver(&self) -> Option<RealtimeEventReceiver> {
625 self.inner.event_rx.lock().take()
626 }
627
628 pub async fn connect(&self) -> Result<(), RealtimeError> {
635 self.inner.connect_once(false).await
636 }
637
638 pub async fn disconnect(&self) -> Result<(), RealtimeError> {
645 self.inner.disconnect().await
646 }
647
648 #[must_use]
650 pub fn is_connected(&self) -> bool {
651 self.inner.is_connected()
652 }
653
654 pub fn session(&self) -> Result<RealtimeSession, RealtimeError> {
659 let lifecycle = self.inner.lifecycle.lock();
660 let Lifecycle::Connected(session) = &*lifecycle else {
661 return Err(RealtimeError::NotConnected);
662 };
663 Ok(RealtimeSession {
664 inner: Arc::downgrade(&self.inner),
665 generation: RealtimeGeneration(session.generation),
666 })
667 }
668
669 pub async fn invoke(
678 &self,
679 target: impl Into<String>,
680 arguments: Vec<Value>,
681 ) -> Result<(), RealtimeError> {
682 self.inner
683 .send_invocation(None, target.into(), arguments)
684 .await
685 }
686
687 pub async fn subscribe_contract_trades(
693 &self,
694 contract: &ContractId,
695 ) -> Result<(), RealtimeError> {
696 if self.inner.hub != Hub::Market {
697 return Err(RealtimeError::WrongHub);
698 }
699 self.session()?.subscribe_contract_trades(contract).await
700 }
701
702 pub async fn unsubscribe_contract_trades(
708 &self,
709 contract: &ContractId,
710 ) -> Result<(), RealtimeError> {
711 if self.inner.hub != Hub::Market {
712 return Err(RealtimeError::WrongHub);
713 }
714 self.session()?.unsubscribe_contract_trades(contract).await
715 }
716
717 pub async fn subscribe_contract_quotes(
723 &self,
724 contract: &ContractId,
725 ) -> Result<(), RealtimeError> {
726 if self.inner.hub != Hub::Market {
727 return Err(RealtimeError::WrongHub);
728 }
729 self.session()?.subscribe_contract_quotes(contract).await
730 }
731
732 pub async fn unsubscribe_contract_quotes(
738 &self,
739 contract: &ContractId,
740 ) -> Result<(), RealtimeError> {
741 if self.inner.hub != Hub::Market {
742 return Err(RealtimeError::WrongHub);
743 }
744 self.session()?.unsubscribe_contract_quotes(contract).await
745 }
746
747 pub async fn subscribe_contract_depth(
753 &self,
754 contract: &ContractId,
755 ) -> Result<(), RealtimeError> {
756 if self.inner.hub != Hub::Market {
757 return Err(RealtimeError::WrongHub);
758 }
759 self.session()?.subscribe_contract_depth(contract).await
760 }
761
762 pub async fn unsubscribe_contract_depth(
768 &self,
769 contract: &ContractId,
770 ) -> Result<(), RealtimeError> {
771 if self.inner.hub != Hub::Market {
772 return Err(RealtimeError::WrongHub);
773 }
774 self.session()?.unsubscribe_contract_depth(contract).await
775 }
776
777 pub async fn subscribe_accounts(&self) -> Result<(), RealtimeError> {
783 if self.inner.hub != Hub::User {
784 return Err(RealtimeError::WrongHub);
785 }
786 self.session()?.subscribe_accounts().await
787 }
788
789 pub async fn unsubscribe_accounts(&self) -> Result<(), RealtimeError> {
795 if self.inner.hub != Hub::User {
796 return Err(RealtimeError::WrongHub);
797 }
798 self.session()?.unsubscribe_accounts().await
799 }
800
801 pub async fn subscribe_orders(&self, account: AccountId) -> Result<(), RealtimeError> {
807 if self.inner.hub != Hub::User {
808 return Err(RealtimeError::WrongHub);
809 }
810 self.session()?.subscribe_orders(account).await
811 }
812
813 pub async fn unsubscribe_orders(&self, account: AccountId) -> Result<(), RealtimeError> {
819 if self.inner.hub != Hub::User {
820 return Err(RealtimeError::WrongHub);
821 }
822 self.session()?.unsubscribe_orders(account).await
823 }
824
825 pub async fn subscribe_positions(&self, account: AccountId) -> Result<(), RealtimeError> {
831 if self.inner.hub != Hub::User {
832 return Err(RealtimeError::WrongHub);
833 }
834 self.session()?.subscribe_positions(account).await
835 }
836
837 pub async fn unsubscribe_positions(&self, account: AccountId) -> Result<(), RealtimeError> {
843 if self.inner.hub != Hub::User {
844 return Err(RealtimeError::WrongHub);
845 }
846 self.session()?.unsubscribe_positions(account).await
847 }
848
849 pub async fn subscribe_trades(&self, account: AccountId) -> Result<(), RealtimeError> {
855 if self.inner.hub != Hub::User {
856 return Err(RealtimeError::WrongHub);
857 }
858 self.session()?.subscribe_trades(account).await
859 }
860
861 pub async fn unsubscribe_trades(&self, account: AccountId) -> Result<(), RealtimeError> {
867 if self.inner.hub != Hub::User {
868 return Err(RealtimeError::WrongHub);
869 }
870 self.session()?.unsubscribe_trades(account).await
871 }
872}
873
874impl Clone for RealtimeClient {
875 fn clone(&self) -> Self {
876 self.inner.client_handles.fetch_add(1, Ordering::Relaxed);
877 Self {
878 inner: Arc::clone(&self.inner),
879 }
880 }
881}
882
883impl Drop for RealtimeClient {
884 fn drop(&mut self) {
885 if self.inner.client_handles.fetch_sub(1, Ordering::AcqRel) == 1 {
886 self.inner.shutdown_now();
887 }
888 }
889}
890
891impl fmt::Debug for RealtimeClient {
892 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
893 formatter
894 .debug_struct("RealtimeClient")
895 .field("hub", &self.inner.hub)
896 .field("endpoints", &self.inner.endpoints)
897 .field("connected", &self.is_connected())
898 .finish_non_exhaustive()
899 }
900}
901
902impl RealtimeInner {
903 fn is_connected(&self) -> bool {
904 matches!(*self.lifecycle.lock(), Lifecycle::Connected(_))
905 }
906
907 fn begin_connect(self: &Arc<Self>, reconnecting: bool) -> Result<ConnectClaim, RealtimeError> {
908 self.begin_connect_with_pre_lock(reconnecting, || {})
909 }
910
911 fn begin_connect_with_pre_lock<F>(
912 self: &Arc<Self>,
913 reconnecting: bool,
914 before_lifecycle_lock: F,
915 ) -> Result<ConnectClaim, RealtimeError>
916 where
917 F: FnOnce(),
918 {
919 before_lifecycle_lock();
920 let mut lifecycle = self.lifecycle.lock();
921 if self.event_flow.has_unacknowledged_gap() {
922 return Err(RealtimeError::TransportGapPending);
923 }
924 if !matches!(*lifecycle, Lifecycle::Disconnected) {
925 return Err(RealtimeError::AlreadyConnected);
926 }
927 if !reconnecting {
928 self.reconnect_enabled.store(true, Ordering::Release);
929 } else if !self.reconnect_enabled.load(Ordering::Acquire) {
930 return Err(RealtimeError::ConnectionCancelled);
931 }
932 let previous = self
933 .generation
934 .fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| {
935 value.checked_add(1)
936 })
937 .map_err(|_| RealtimeError::IdentifierCapacity)?;
938 let generation = previous
939 .checked_add(1)
940 .ok_or(RealtimeError::IdentifierCapacity)?;
941 let cancellation = self.owner_cancel.child_token();
942 *lifecycle = Lifecycle::Connecting {
943 generation,
944 cancellation: cancellation.clone(),
945 };
946 drop(lifecycle);
947 self.lifecycle_changed.notify_waiters();
948 Ok(ConnectClaim {
949 inner: Arc::downgrade(self),
950 generation,
951 cancellation,
952 complete: false,
953 })
954 }
955
956 async fn connect_once(self: &Arc<Self>, reconnecting: bool) -> Result<(), RealtimeError> {
957 let mut claim = self.begin_connect(reconnecting)?;
958 let url = self.connection_url()?;
959 let mut stream = tokio::select! {
960 () = claim.cancellation.cancelled() => {
961 return Err(RealtimeError::ConnectionCancelled);
962 }
963 result = tokio::time::timeout(
964 CONNECT_TIMEOUT,
965 upgrade_websocket(&self.http, url),
966 ) => {
967 match result {
968 Ok(result) => result?,
969 Err(_) => return Err(RealtimeError::ConnectionTimedOut),
970 }
971 }
972 };
973 let handshake_tail = tokio::select! {
974 () = claim.cancellation.cancelled() => {
975 return Err(RealtimeError::ConnectionCancelled);
976 }
977 result = tokio::time::timeout(HANDSHAKE_TIMEOUT, negotiate_handshake(&mut stream)) => {
978 match result {
979 Ok(result) => result?,
980 Err(_) => return Err(RealtimeError::Handshake("response timed out")),
981 }
982 }
983 };
984
985 let (write, read) = stream.split();
986 let (writer, writer_rx) = mpsc::channel(self.config.writer_capacity);
987 let start = CancellationToken::new();
988 let session_cancellation = claim.cancellation.child_token();
989 let writer_task = tokio::spawn(run_writer(
990 Arc::downgrade(self),
991 claim.generation,
992 write,
993 writer_rx,
994 session_cancellation.clone(),
995 start.clone(),
996 ));
997 let reader_task = tokio::spawn(run_reader(
998 Arc::downgrade(self),
999 claim.generation,
1000 read,
1001 handshake_tail,
1002 session_cancellation.clone(),
1003 start.clone(),
1004 ));
1005 let session = Session {
1006 runtime: tokio::runtime::Handle::current(),
1007 generation: claim.generation,
1008 writer,
1009 cancellation: session_cancellation,
1010 reader_task,
1011 writer_task,
1012 };
1013 let event = if reconnecting {
1014 RealtimeEvent::Reconnected
1015 } else {
1016 RealtimeEvent::Connected
1017 };
1018 self.install_ready(claim.generation, session, event)?;
1019 self.record_activity(claim.generation);
1020 start.cancel();
1021 self.start_watchdog();
1022 claim.complete();
1023 Ok(())
1024 }
1025
1026 fn install_ready(
1027 self: &Arc<Self>,
1028 generation: u64,
1029 session: Session,
1030 event: RealtimeEvent,
1031 ) -> Result<(), RealtimeError> {
1032 let mut lifecycle = self.lifecycle.lock();
1033 let can_install = matches!(
1034 &*lifecycle,
1035 Lifecycle::Connecting {
1036 generation: active,
1037 cancellation,
1038 } if *active == generation && !cancellation.is_cancelled()
1039 );
1040 if !can_install {
1041 return Err(RealtimeError::ConnectionCancelled);
1042 }
1043 self.event_flow.start_generation(generation)?;
1044 *lifecycle = Lifecycle::Connected(session);
1045 let published = self.publish(generation, event, 0);
1046 drop(lifecycle);
1047 self.lifecycle_changed.notify_waiters();
1048 match published {
1049 Ok(PublishOutcome::Published) => Ok(()),
1050 Ok(PublishOutcome::StaleGeneration | PublishOutcome::GapFenced) => {
1051 self.end_generation(generation);
1052 Err(RealtimeError::ConnectionCancelled)
1053 }
1054 Err(error) => {
1055 self.end_generation(generation);
1056 Err(error)
1057 }
1058 }
1059 }
1060
1061 fn cancel_connect(self: &Arc<Self>, generation: u64) {
1062 self.end_generation(generation);
1063 }
1064
1065 fn begin_disconnect(&self) -> DisconnectAction {
1066 let mut lifecycle = self.lifecycle.lock();
1067 self.reconnect_enabled.store(false, Ordering::Release);
1068 self.stop_watchdog();
1069 let previous = std::mem::replace(&mut *lifecycle, Lifecycle::Disconnected);
1070 let action = match previous {
1071 Lifecycle::Disconnected => DisconnectAction::None,
1072 Lifecycle::Connecting {
1073 generation,
1074 cancellation,
1075 } => {
1076 cancellation.cancel();
1077 *lifecycle = Lifecycle::Closing {
1078 generation,
1079 was_connected: false,
1080 };
1081 DisconnectAction::Wait(generation)
1082 }
1083 Lifecycle::Connected(session) => {
1084 let generation = session.generation;
1085 *lifecycle = Lifecycle::Closing {
1086 generation,
1087 was_connected: true,
1088 };
1089 DisconnectAction::Close(session)
1090 }
1091 closing @ Lifecycle::Closing { generation, .. } => {
1092 *lifecycle = closing;
1093 DisconnectAction::Wait(generation)
1094 }
1095 };
1096 drop(lifecycle);
1097 self.lifecycle_changed.notify_waiters();
1098 action
1099 }
1100
1101 async fn disconnect(self: &Arc<Self>) -> Result<(), RealtimeError> {
1102 match self.begin_disconnect() {
1103 DisconnectAction::None => Ok(()),
1104 DisconnectAction::Wait(generation) => {
1105 let waited = tokio::time::timeout(
1106 CLOSE_TIMEOUT,
1107 self.wait_until_generation_ends(generation),
1108 )
1109 .await;
1110 if waited.is_err() {
1111 Err(RealtimeError::Close)
1112 } else {
1113 Ok(())
1114 }
1115 }
1116 DisconnectAction::Close(session) => {
1117 let generation = session.generation;
1118 self.retire_session(session, true);
1119 self.wait_until_generation_ends(generation).await;
1120 if self.failed_close.load(Ordering::Acquire) == generation {
1121 Err(RealtimeError::Close)
1122 } else {
1123 Ok(())
1124 }
1125 }
1126 }
1127 }
1128
1129 async fn wait_until_generation_ends(&self, generation: u64) {
1130 loop {
1131 let changed = self.lifecycle_changed.notified();
1132 tokio::pin!(changed);
1133 let _ = changed.as_mut().enable();
1134 let is_active = match &*self.lifecycle.lock() {
1135 Lifecycle::Connecting {
1136 generation: active, ..
1137 }
1138 | Lifecycle::Closing {
1139 generation: active, ..
1140 } => *active == generation,
1141 Lifecycle::Connected(session) => session.generation == generation,
1142 Lifecycle::Disconnected => false,
1143 };
1144 if !is_active {
1145 return;
1146 }
1147 changed.await;
1148 }
1149 }
1150
1151 fn finish_closing(&self, generation: u64) {
1152 let mut lifecycle = self.lifecycle.lock();
1153 let should_publish = match &*lifecycle {
1154 Lifecycle::Closing {
1155 generation: active,
1156 was_connected,
1157 } if *active == generation => *was_connected,
1158 _ => return,
1159 };
1160 self.finish_event_generation_locked(generation, should_publish);
1161 *lifecycle = Lifecycle::Disconnected;
1162 drop(lifecycle);
1163 self.fail_pending_generation(generation);
1164 self.lifecycle_changed.notify_waiters();
1165 }
1166
1167 fn end_generation(self: &Arc<Self>, generation: u64) {
1168 let mut lifecycle = self.lifecycle.lock();
1169 let previous = std::mem::replace(&mut *lifecycle, Lifecycle::Disconnected);
1170 match previous {
1171 Lifecycle::Connected(session) if session.generation == generation => {
1172 *lifecycle = Lifecycle::Closing {
1173 generation,
1174 was_connected: true,
1175 };
1176 drop(lifecycle);
1177 self.retire_session(session, false);
1178 }
1179 Lifecycle::Connecting {
1180 generation: active,
1181 cancellation,
1182 } if active == generation => {
1183 cancellation.cancel();
1184 drop(lifecycle);
1185 self.lifecycle_changed.notify_waiters();
1186 }
1187 Lifecycle::Closing {
1188 generation: active,
1189 was_connected: false,
1190 } if active == generation => {
1191 drop(lifecycle);
1192 self.lifecycle_changed.notify_waiters();
1193 }
1194 other => {
1195 *lifecycle = other;
1196 }
1197 }
1198 }
1199
1200 fn retire_session(self: &Arc<Self>, mut session: Session, graceful: bool) {
1203 let generation = session.generation;
1204 if !graceful {
1205 session.cancellation.cancel();
1206 }
1207 self.fail_pending_generation(generation);
1208 let inner = Arc::clone(self);
1209 let mut slot = self.retirement_task.lock();
1210 let runtime = session.runtime.clone();
1213 *slot = Some(runtime.spawn(async move {
1214 if graceful && send_queued(&session.writer, Message::Close(None)).is_err() {
1217 session.cancellation.cancel();
1218 inner.failed_close.store(generation, Ordering::Release);
1219 }
1220 if !join_socket_tasks(&mut session).await {
1221 inner.failed_close.store(generation, Ordering::Release);
1222 }
1223 inner.finish_closing(generation);
1224 }));
1225 }
1226
1227 fn shutdown_now(self: &Arc<Self>) {
1228 self.owner_cancel.cancel();
1229 self.reconnect_enabled.store(false, Ordering::Release);
1230 self.stop_watchdog();
1231 let generation = self.lifecycle.lock().generation();
1232 if let Some(generation) = generation {
1233 self.end_generation(generation);
1234 }
1235 self.pending.lock().clear();
1236 }
1237
1238 fn finish_event_generation_locked(&self, generation: u64, publish_disconnected: bool) {
1241 if publish_disconnected {
1242 let _ = self.publish(generation, RealtimeEvent::Disconnected, 0);
1243 }
1244 self.event_flow.finish_generation(generation);
1245 }
1246
1247 fn start_watchdog(self: &Arc<Self>) {
1248 let mut watchdog = self.watchdog_task.lock();
1249 if !self.reconnect_enabled.load(Ordering::Acquire) || self.owner_cancel.is_cancelled() {
1250 return;
1251 }
1252 if watchdog.as_ref().is_some_and(|task| !task.is_finished()) {
1253 return;
1254 }
1255 if let Some(task) = watchdog.take() {
1256 task.abort();
1257 }
1258 let weak = Arc::downgrade(self);
1259 let cancellation = self.owner_cancel.clone();
1260 *watchdog = Some(tokio::spawn(async move {
1261 let mut ticker = tokio::time::interval(WATCHDOG_INTERVAL);
1262 ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
1263 loop {
1264 tokio::select! {
1265 biased;
1266 () = cancellation.cancelled() => break,
1267 _ = ticker.tick() => {
1268 let Some(inner) = weak.upgrade() else {
1269 break;
1270 };
1271 inner.watchdog_tick().await;
1272 }
1273 }
1274 }
1275 }));
1276 }
1277
1278 fn stop_watchdog(&self) {
1279 if let Some(task) = self.watchdog_task.lock().take() {
1280 task.abort();
1281 }
1282 }
1283
1284 async fn watchdog_tick(self: &Arc<Self>) {
1285 if !self.reconnect_enabled.load(Ordering::Acquire)
1286 || self.event_flow.has_unacknowledged_gap()
1287 || self.owner_cancel.is_cancelled()
1288 {
1289 return;
1290 }
1291 let disconnected = matches!(*self.lifecycle.lock(), Lifecycle::Disconnected);
1292 if disconnected
1293 && let Err(error) = self.connect_once(true).await
1294 && !matches!(
1295 error,
1296 RealtimeError::ConnectionCancelled | RealtimeError::AlreadyConnected
1297 )
1298 {
1299 tracing::warn!(%error, hub = ?self.hub, "ProjectX real-time reconnect failed");
1300 }
1301 }
1302
1303 fn connection_url(&self) -> Result<url::Url, RealtimeError> {
1304 let token = self
1305 .token
1306 .snapshot()
1307 .filter(|token| !token.trim().is_empty())
1308 .ok_or(RealtimeError::MissingAuthToken)?;
1309 let mut url = self
1310 .endpoints
1311 .hub_url(self.hub.path())
1312 .map_err(RealtimeError::Endpoint)?;
1313 url.query_pairs_mut().append_pair("access_token", &token);
1314 Ok(url)
1315 }
1316
1317 async fn send_invocation(
1318 self: &Arc<Self>,
1319 expected: Option<RealtimeGeneration>,
1320 target: String,
1321 arguments: Vec<Value>,
1322 ) -> Result<(), RealtimeError> {
1323 let invocation_id = self
1324 .request_counter
1325 .fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| {
1326 value.checked_add(1)
1327 })
1328 .map_err(|_| RealtimeError::IdentifierCapacity)?
1329 .to_string();
1330 let message = encode_invocation(&invocation_id, &target, &arguments)?;
1331 let (reply_tx, reply_rx) = oneshot::channel();
1332 let (generation, writer) = self.register_invocation(expected, &invocation_id, reply_tx)?;
1333 let _pending = PendingGuard {
1334 pending: Arc::clone(&self.pending),
1335 invocation_id: invocation_id.clone(),
1336 generation,
1337 };
1338 send_queued(&writer, message)?;
1339
1340 match tokio::time::timeout(self.config.invocation_timeout, reply_rx).await {
1341 Ok(Ok(Ok(()))) => Ok(()),
1342 Ok(Ok(Err(()))) => Err(RealtimeError::InvocationRejected { target }),
1343 Ok(Err(_)) => Err(RealtimeError::InvocationSessionEnded { target }),
1344 Err(_) => Err(RealtimeError::InvocationTimedOut { target }),
1345 }
1346 }
1347
1348 fn register_invocation(
1349 &self,
1350 expected: Option<RealtimeGeneration>,
1351 invocation_id: &str,
1352 reply: PendingInvocation,
1353 ) -> Result<(u64, mpsc::Sender<Message>), RealtimeError> {
1354 let lifecycle = self.lifecycle.lock();
1355 let Lifecycle::Connected(session) = &*lifecycle else {
1356 return Err(RealtimeError::NotConnected);
1357 };
1358 if expected.is_some_and(|expected| expected.0 != session.generation) {
1359 return Err(RealtimeError::StaleGeneration);
1360 }
1361 let mut pending = self.pending.lock();
1362 if pending.len() >= self.config.pending_capacity {
1363 return Err(RealtimeError::PendingInvocationCapacity);
1364 }
1365 pending.insert(
1366 invocation_id.to_owned(),
1367 PendingEntry {
1368 generation: session.generation,
1369 reply,
1370 },
1371 );
1372 Ok((session.generation, session.writer.clone()))
1373 }
1374
1375 async fn process_text(
1376 &self,
1377 generation: u64,
1378 text: &str,
1379 ) -> Result<ProcessOutcome, RealtimeError> {
1380 let mut records = text.split(SIGNALR_TERMINATOR).enumerate().peekable();
1381 while let Some((index, frame)) = records.next() {
1384 if index != 0 && index % COALESCED_RECORD_QUANTUM == 0 {
1385 tokio::task::yield_now().await;
1386 }
1387 if records.peek().is_none() {
1388 if !frame.is_empty() {
1389 self.event_flow.mark_gap(generation);
1390 }
1391 break;
1392 }
1393 match self.process_record(generation, frame) {
1394 Ok(ProcessOutcome::Continue) => {}
1395 Ok(ProcessOutcome::Close) => return Ok(ProcessOutcome::Close),
1396 Err(RealtimeError::Decode(_) | RealtimeError::Protocol(_)) => {
1397 self.event_flow.mark_gap(generation);
1398 }
1399 Err(error) => return Err(error),
1400 }
1401 }
1402 Ok(ProcessOutcome::Continue)
1403 }
1404
1405 fn process_record(
1406 &self,
1407 generation: u64,
1408 frame: &str,
1409 ) -> Result<ProcessOutcome, RealtimeError> {
1410 let value: Value = serde_json::from_str(frame).map_err(RealtimeError::Decode)?;
1411 let message_type = value
1412 .get("type")
1413 .ok_or(RealtimeError::Protocol("message type is missing"))?
1414 .as_u64()
1415 .ok_or(RealtimeError::Protocol(
1416 "message type was not an unsigned integer",
1417 ))?;
1418 match message_type {
1419 1 => {
1420 let invocation = SignalRInvocation::from_json(frame)?.ok_or(
1421 RealtimeError::Protocol("type-1 frame was not an invocation"),
1422 )?;
1423 let weight = frame.len();
1424 if self.publish(generation, RealtimeEvent::Invocation(invocation), weight)?
1425 == PublishOutcome::StaleGeneration
1426 {
1427 return Ok(ProcessOutcome::Close);
1428 }
1429 }
1430 3 => {
1431 let (invocation_id, result) = completion(&value)?;
1432 let reply = {
1433 let mut pending = self.pending.lock();
1434 if pending
1435 .get(invocation_id)
1436 .is_some_and(|entry| entry.generation == generation)
1437 {
1438 pending.remove(invocation_id).map(|entry| entry.reply)
1439 } else {
1440 None
1441 }
1442 };
1443 if let Some(reply) = reply {
1444 let _ = reply.send(result);
1445 }
1446 }
1447 6 => {}
1448 7 => {
1449 if value.get("error").is_some_and(|error| !error.is_string()) {
1450 return Err(RealtimeError::Protocol("close error was not a string"));
1451 }
1452 let _provider_hint = match value.get("allowReconnect") {
1453 Some(flag) => flag.as_bool().ok_or(RealtimeError::Protocol(
1454 "close allowReconnect flag was not a boolean",
1455 ))?,
1456 None => false,
1457 };
1458 return Ok(ProcessOutcome::Close);
1459 }
1460 _ => {
1461 let weight = frame.len();
1462 if self.publish(generation, RealtimeEvent::Message(value), weight)?
1463 == PublishOutcome::StaleGeneration
1464 {
1465 return Ok(ProcessOutcome::Close);
1466 }
1467 }
1468 }
1469 Ok(ProcessOutcome::Continue)
1470 }
1471
1472 fn publish(
1473 &self,
1474 generation: u64,
1475 event: RealtimeEvent,
1476 weight: usize,
1477 ) -> Result<PublishOutcome, RealtimeError> {
1478 let weight = weight.max(EVENT_BASE_WEIGHT);
1479 match self
1480 .event_flow
1481 .publish(&self.event_tx, generation, event, weight)
1482 {
1483 Err(RealtimeError::EventReceiverClosed) => {
1484 self.reconnect_enabled.store(false, Ordering::Release);
1485 self.stop_watchdog();
1486 Err(RealtimeError::EventReceiverClosed)
1487 }
1488 result => result,
1489 }
1490 }
1491
1492 fn fail_pending_generation(&self, generation: u64) {
1493 self.pending
1494 .lock()
1495 .retain(|_, entry| entry.generation != generation);
1496 }
1497
1498 fn acknowledge_probe(&self, payload: &[u8]) {
1499 let Ok(payload) = <[u8; 16]>::try_from(payload) else {
1500 return;
1501 };
1502 let (left, right) = payload.split_at(8);
1503 let (Ok(left), Ok(right)) = (left.try_into(), right.try_into()) else {
1504 return;
1505 };
1506 let answer = (u64::from_be_bytes(left), u64::from_be_bytes(right));
1507 let mut probe = self.socket_probe.lock();
1508 if *probe == Some(answer) {
1509 *probe = None;
1510 }
1511 }
1512
1513 fn record_activity(&self, generation: u64) {
1514 let lifecycle = self.lifecycle.lock();
1515 if matches!(
1516 &*lifecycle,
1517 Lifecycle::Connected(session) if session.generation == generation
1518 ) {
1519 *self.last_activity.lock() = Instant::now();
1520 }
1521 }
1522}
1523
1524impl Drop for RealtimeInner {
1525 fn drop(&mut self) {
1526 self.owner_cancel.cancel();
1527 if let Some(task) = self.watchdog_task.get_mut().take() {
1528 task.abort();
1529 }
1530 match std::mem::replace(self.lifecycle.get_mut(), Lifecycle::Disconnected) {
1531 Lifecycle::Connecting { cancellation, .. } => cancellation.cancel(),
1532 Lifecycle::Connected(session) => drop(session),
1533 Lifecycle::Disconnected | Lifecycle::Closing { .. } => {}
1534 }
1535 }
1536}
1537
1538async fn upgrade_websocket(
1539 http: &reqwest::Client,
1540 mut url: url::Url,
1541) -> Result<WebSocketStream<reqwest::Upgraded>, RealtimeError> {
1542 let http_scheme = match url.scheme() {
1543 "wss" => "https",
1544 "ws" => "http",
1545 _ => return Err(RealtimeError::Transport),
1546 };
1547 url.set_scheme(http_scheme)
1548 .map_err(|()| RealtimeError::Transport)?;
1549
1550 let key = websocket_key()?;
1551 let expected_accept = websocket_accept(&key);
1552 let response = http
1553 .get(url)
1554 .version(Version::HTTP_11)
1555 .header(header::CONNECTION, "Upgrade")
1556 .header(header::UPGRADE, "websocket")
1557 .header(header::SEC_WEBSOCKET_VERSION, "13")
1558 .header(header::SEC_WEBSOCKET_KEY, key)
1559 .send()
1560 .await
1561 .map_err(|_| RealtimeError::Transport)?;
1562 validate_websocket_upgrade(
1563 response.status(),
1564 response.version(),
1565 response.headers(),
1566 &expected_accept,
1567 )?;
1568 let upgraded = response
1569 .upgrade()
1570 .await
1571 .map_err(|_| RealtimeError::Transport)?;
1572 Ok(WebSocketStream::from_raw_socket(upgraded, Role::Client, Some(websocket_config())).await)
1573}
1574
1575fn websocket_key() -> Result<String, RealtimeError> {
1576 let mut nonce = [0_u8; 16];
1577 SysRng
1578 .try_fill_bytes(&mut nonce)
1579 .map_err(|_| RealtimeError::Transport)?;
1580 Ok(BASE64.encode(&nonce))
1581}
1582
1583fn websocket_accept(key: &str) -> String {
1584 let mut digest = Sha1::new();
1585 digest.update(key.as_bytes());
1586 digest.update(WEBSOCKET_GUID);
1587 BASE64.encode(&digest.finalize())
1588}
1589
1590fn validate_websocket_upgrade(
1591 status: StatusCode,
1592 version: Version,
1593 headers: &header::HeaderMap,
1594 expected_accept: &str,
1595) -> Result<(), RealtimeError> {
1596 let mut accept_values = headers.get_all(header::SEC_WEBSOCKET_ACCEPT).iter();
1597 let accept_matches = accept_values
1598 .next()
1599 .is_some_and(|value| value.as_bytes() == expected_accept.as_bytes())
1600 && accept_values.next().is_none();
1601 if status != StatusCode::SWITCHING_PROTOCOLS
1602 || version != Version::HTTP_11
1603 || !header_contains_token(headers, &header::CONNECTION, "upgrade")
1604 || !header_contains_token(headers, &header::UPGRADE, "websocket")
1605 || !accept_matches
1606 || headers.contains_key(header::SEC_WEBSOCKET_EXTENSIONS)
1607 || headers.contains_key(header::SEC_WEBSOCKET_PROTOCOL)
1608 {
1609 return Err(RealtimeError::Transport);
1610 }
1611 Ok(())
1612}
1613
1614fn header_contains_token(
1615 headers: &header::HeaderMap,
1616 name: &header::HeaderName,
1617 expected: &str,
1618) -> bool {
1619 headers.get_all(name).iter().any(|value| {
1620 value.to_str().is_ok_and(|value| {
1621 value
1622 .split(',')
1623 .any(|token| token.trim().eq_ignore_ascii_case(expected))
1624 })
1625 })
1626}
1627
1628fn websocket_config() -> WebSocketConfig {
1629 WebSocketConfig::default()
1630 .read_buffer_size(WEBSOCKET_READ_BUFFER_SIZE)
1631 .write_buffer_size(WEBSOCKET_WRITE_BUFFER_SIZE)
1632 .max_message_size(None)
1635 .max_frame_size(None)
1636}
1637
1638fn encode_invocation(
1639 invocation_id: &str,
1640 target: &str,
1641 arguments: &[Value],
1642) -> Result<Message, RealtimeError> {
1643 let payload = OutboundInvocation {
1644 message_type: 1,
1645 invocation_id,
1646 target,
1647 arguments,
1648 };
1649 let mut text = serde_json::to_string(&payload).map_err(RealtimeError::Encode)?;
1650 text.push(SIGNALR_TERMINATOR);
1651 Ok(Message::Text(text.into()))
1652}
1653
1654fn send_queued(writer: &mpsc::Sender<Message>, message: Message) -> Result<(), RealtimeError> {
1655 writer.try_send(message).map_err(|error| match error {
1656 mpsc::error::TrySendError::Full(_) => RealtimeError::SendQueueFull,
1657 mpsc::error::TrySendError::Closed(_) => RealtimeError::SendClosed,
1658 })
1659}
1660
1661async fn join_socket_tasks(session: &mut Session) -> bool {
1662 let deadline = tokio::time::sleep(CLOSE_TIMEOUT);
1663 tokio::pin!(deadline);
1664 let mut reader_done = false;
1665 let mut writer_done = false;
1666 loop {
1667 if reader_done && writer_done {
1668 return true;
1669 }
1670 tokio::select! {
1671 _ = &mut session.reader_task, if !reader_done => reader_done = true,
1672 _ = &mut session.writer_task, if !writer_done => writer_done = true,
1673 () = &mut deadline => break,
1674 }
1675 }
1676 if !reader_done {
1677 session.reader_task.abort();
1678 let _ = (&mut session.reader_task).await;
1679 }
1680 if !writer_done {
1681 session.writer_task.abort();
1682 let _ = (&mut session.writer_task).await;
1683 }
1684 false
1685}
1686
1687async fn negotiate_handshake<S>(stream: &mut S) -> Result<Option<String>, RealtimeError>
1688where
1689 S: futures_util::Stream<Item = Result<Message, TungsteniteError>>
1690 + futures_util::Sink<Message, Error = TungsteniteError>
1691 + Unpin,
1692{
1693 let payload = format!(
1694 "{}{}",
1695 serde_json::json!({"protocol":"json","version":1}),
1696 SIGNALR_TERMINATOR
1697 );
1698 stream
1699 .send(Message::Text(payload.into()))
1700 .await
1701 .map_err(|_| RealtimeError::Transport)?;
1702 loop {
1703 match stream.next().await {
1704 Some(Ok(Message::Text(text))) => return validate_handshake(text.as_ref()),
1705 Some(Ok(Message::Binary(bytes))) => {
1706 let text = std::str::from_utf8(bytes.as_ref())
1707 .map_err(|_| RealtimeError::Handshake("response was not UTF-8"))?;
1708 return validate_handshake(text);
1709 }
1710 Some(Ok(Message::Ping(_) | Message::Pong(_) | Message::Frame(_))) => {}
1713 Some(Ok(Message::Close(_))) | None => {
1714 return Err(RealtimeError::Handshake(
1715 "connection closed before the response",
1716 ));
1717 }
1718 Some(Err(_)) => return Err(RealtimeError::Transport),
1719 }
1720 }
1721}
1722
1723async fn run_writer<S>(
1724 inner: Weak<RealtimeInner>,
1725 generation: u64,
1726 mut write: S,
1727 mut messages: mpsc::Receiver<Message>,
1728 cancellation: CancellationToken,
1729 start: CancellationToken,
1730) -> Result<(), RealtimeError>
1731where
1732 S: futures_util::Sink<Message, Error = TungsteniteError> + Unpin,
1733{
1734 tokio::select! {
1735 biased;
1736 () = cancellation.cancelled() => {
1737 let _ = tokio::time::timeout(
1738 Duration::from_secs(1),
1739 write.send(Message::Close(None)),
1740 )
1741 .await;
1742 return Ok(());
1743 }
1744 () = start.cancelled() => {}
1745 }
1746 let keepalive = tokio::time::sleep(CLIENT_KEEPALIVE_INTERVAL);
1747 tokio::pin!(keepalive);
1748 let probe_tick = tokio::time::sleep(SOCKET_PROBE_INTERVAL);
1749 tokio::pin!(probe_tick);
1750 let mut probe_id = 0_u64;
1751 let result = loop {
1752 let message = tokio::select! {
1753 biased;
1754 () = cancellation.cancelled() => {
1755 let _ = tokio::time::timeout(
1756 Duration::from_secs(1),
1757 write.send(Message::Close(None)),
1758 )
1759 .await;
1760 break Ok(());
1761 },
1762 () = &mut probe_tick => {
1763 let Some(inner) = inner.upgrade() else { break Ok(()); };
1764 let mut probe = inner.socket_probe.lock();
1765 if probe_id != 0 && *probe == Some((generation, probe_id)) {
1766 break Err(RealtimeError::PingTimedOut);
1767 }
1768 let Some(next) = probe_id.checked_add(1) else { break Err(RealtimeError::IdentifierCapacity); };
1769 probe_id = next;
1770 *probe = Some((generation, probe_id));
1771 let mut payload = Vec::with_capacity(16);
1772 payload.extend_from_slice(&generation.to_be_bytes());
1773 payload.extend_from_slice(&probe_id.to_be_bytes());
1774 probe_tick.as_mut().reset(Instant::now() + SOCKET_PROBE_INTERVAL);
1775 Message::Ping(payload.into())
1776 }
1777 message = messages.recv() => {
1778 match message {
1779 Some(message) => message,
1780 None => break Err(RealtimeError::SendClosed),
1781 }
1782 }
1783 () = &mut keepalive => Message::Text(SIGNALR_PING.into()),
1784 };
1785 let is_close = matches!(message, Message::Close(_));
1786 let is_signalr = matches!(message, Message::Text(_) | Message::Binary(_));
1787 if write.send(message).await.is_err() {
1788 break Err(RealtimeError::Transport);
1789 }
1790 if is_close {
1791 break Ok(());
1792 }
1793 if is_signalr {
1794 keepalive
1795 .as_mut()
1796 .reset(Instant::now() + CLIENT_KEEPALIVE_INTERVAL);
1797 }
1798 };
1799 if result.is_err() && !cancellation.is_cancelled() {
1800 cancellation.cancel();
1801 if let Some(inner) = inner.upgrade() {
1802 inner.end_generation(generation);
1803 }
1804 }
1805 result
1806}
1807
1808async fn run_reader<S>(
1809 inner: Weak<RealtimeInner>,
1810 generation: u64,
1811 mut read: S,
1812 handshake_tail: Option<String>,
1813 cancellation: CancellationToken,
1814 start: CancellationToken,
1815) -> Result<(), RealtimeError>
1816where
1817 S: futures_util::Stream<Item = Result<Message, TungsteniteError>> + Unpin,
1818{
1819 tokio::select! {
1820 biased;
1821 () = cancellation.cancelled() => return Ok(()),
1822 () = start.cancelled() => {}
1823 }
1824 let result = async {
1825 if let Some(tail) = handshake_tail {
1826 let Some(inner) = inner.upgrade() else {
1827 return Ok(());
1828 };
1829 if inner.process_text(generation, &tail).await? == ProcessOutcome::Close {
1830 return Ok(());
1831 }
1832 }
1833 loop {
1834 let message = tokio::select! {
1835 biased;
1836 () = cancellation.cancelled() => return Ok(()),
1837 message = read.next() => message,
1838 };
1839 match message {
1840 Some(Ok(Message::Text(text))) => {
1841 let Some(inner) = inner.upgrade() else {
1842 return Ok(());
1843 };
1844 let outcome = inner.process_text(generation, text.as_ref()).await?;
1845 inner.record_activity(generation);
1846 if outcome == ProcessOutcome::Close {
1847 return Ok(());
1848 }
1849 }
1850 Some(Ok(Message::Binary(bytes))) => {
1851 let Some(inner) = inner.upgrade() else {
1852 return Ok(());
1853 };
1854 let Ok(text) = std::str::from_utf8(bytes.as_ref()) else {
1855 inner.event_flow.mark_gap(generation);
1856 continue;
1857 };
1858 let outcome = inner.process_text(generation, text).await?;
1859 inner.record_activity(generation);
1860 if outcome == ProcessOutcome::Close {
1861 return Ok(());
1862 }
1863 }
1864 Some(Ok(Message::Ping(_))) => {
1865 let Some(inner) = inner.upgrade() else {
1866 return Ok(());
1867 };
1868 inner.record_activity(generation);
1869 }
1871 Some(Ok(Message::Close(_))) | None => return Ok(()),
1872 Some(Err(_)) => return Err(RealtimeError::Transport),
1873 Some(Ok(Message::Pong(payload))) => {
1874 let Some(inner) = inner.upgrade() else {
1875 return Ok(());
1876 };
1877 inner.acknowledge_probe(&payload);
1878 inner.record_activity(generation);
1879 }
1880 Some(Ok(Message::Frame(_))) => {}
1881 }
1882 }
1883 }
1884 .await;
1885 if !cancellation.is_cancelled() {
1886 cancellation.cancel();
1887 if let Some(inner) = inner.upgrade() {
1888 inner.end_generation(generation);
1889 }
1890 }
1891 result
1892}
1893
1894fn validate_handshake(text: &str) -> Result<Option<String>, RealtimeError> {
1895 let (payload, tail) = text
1896 .split_once(SIGNALR_TERMINATOR)
1897 .ok_or(RealtimeError::Handshake("response frame was incomplete"))?;
1898 let response: Value = serde_json::from_str(payload)
1899 .map_err(|_| RealtimeError::Handshake("response was invalid JSON"))?;
1900 if response
1901 .get("error")
1902 .and_then(Value::as_str)
1903 .is_some_and(|detail| !detail.trim().is_empty())
1904 {
1905 return Err(RealtimeError::Handshake("provider rejected the handshake"));
1906 }
1907 if response.as_object().is_some_and(serde_json::Map::is_empty) {
1908 Ok((!tail.is_empty()).then(|| tail.to_owned()))
1909 } else {
1910 Err(RealtimeError::Handshake(
1911 "provider returned an unexpected response shape",
1912 ))
1913 }
1914}
1915
1916fn completion(value: &Value) -> Result<(&str, Result<(), ()>), RealtimeError> {
1917 let object = value
1918 .as_object()
1919 .ok_or(RealtimeError::Protocol("completion was not an object"))?;
1920 let invocation_id =
1921 object
1922 .get("invocationId")
1923 .and_then(Value::as_str)
1924 .ok_or(RealtimeError::Protocol(
1925 "completion invocationId was not a string",
1926 ))?;
1927 if object.contains_key("error") && object.contains_key("result") {
1928 return Err(RealtimeError::Protocol(
1929 "completion contained both error and result",
1930 ));
1931 }
1932 let result = match object.get("error") {
1933 Some(Value::String(_)) => Err(()),
1934 Some(_) => {
1935 return Err(RealtimeError::Protocol("completion error was not a string"));
1936 }
1937 None => Ok(()),
1938 };
1939 Ok((invocation_id, result))
1940}
1941
1942#[cfg(test)]
1943mod tests {
1944 use serde_json::json;
1945
1946 use super::*;
1947
1948 fn fixture_realtime() -> RealtimeClient {
1949 let credentials = crate::Credentials::new("user", "key")
1950 .unwrap_or_else(|error| panic!("fixture credentials must be valid: {error}"));
1951 let client = crate::Client::builder(credentials)
1952 .build()
1953 .unwrap_or_else(|error| panic!("fixture client must build: {error}"));
1954 client.realtime(Hub::Market)
1955 }
1956
1957 fn install_connected_generation(inner: &Arc<RealtimeInner>, generation: u64) {
1958 let (writer, _writer_rx) = mpsc::channel(1);
1959 let session = Session {
1960 runtime: tokio::runtime::Handle::current(),
1961 generation,
1962 writer,
1963 cancellation: CancellationToken::new(),
1964 reader_task: tokio::spawn(std::future::pending::<Result<(), RealtimeError>>()),
1965 writer_task: tokio::spawn(std::future::pending::<Result<(), RealtimeError>>()),
1966 };
1967 let mut lifecycle = inner.lifecycle.lock();
1968 inner
1969 .event_flow
1970 .start_generation(generation)
1971 .unwrap_or_else(|error| panic!("fixture generation must start: {error}"));
1972 inner.generation.store(generation, Ordering::Release);
1973 *lifecycle = Lifecycle::Connected(session);
1974 }
1975
1976 fn install_paused_connected_generation(
1977 inner: &Arc<RealtimeInner>,
1978 generation: u64,
1979 ) -> mpsc::Receiver<Message> {
1980 let (writer, writer_rx) = mpsc::channel(1);
1981 let session = Session {
1982 runtime: tokio::runtime::Handle::current(),
1983 generation,
1984 writer,
1985 cancellation: CancellationToken::new(),
1986 reader_task: tokio::spawn(std::future::pending::<Result<(), RealtimeError>>()),
1987 writer_task: tokio::spawn(std::future::pending::<Result<(), RealtimeError>>()),
1988 };
1989 let mut lifecycle = inner.lifecycle.lock();
1990 inner
1991 .event_flow
1992 .start_generation(generation)
1993 .unwrap_or_else(|error| panic!("fixture generation must start: {error}"));
1994 inner.generation.store(generation, Ordering::Release);
1995 *lifecycle = Lifecycle::Connected(session);
1996 writer_rx
1997 }
1998
1999 #[test]
2000 fn invocation_extracts_market_contract_and_payload() {
2001 let value = json!({
2002 "type": 1,
2003 "target": "GatewayTrade",
2004 "arguments": ["CON.F.US.MNQ.M26", {"price": 1.25}],
2005 });
2006 let invocation = SignalRInvocation::from_value(&value)
2007 .and_then(|value| value.ok_or(RealtimeError::Protocol("missing invocation")))
2008 .unwrap_or_else(|error| panic!("fixture invocation must decode: {error}"));
2009 assert_eq!(invocation.target(), "GatewayTrade");
2010 assert_eq!(
2011 invocation.contract_id().map(ContractId::as_str),
2012 Some("CON.F.US.MNQ.M26")
2013 );
2014 assert_eq!(invocation.payload(), &json!({"price": 1.25}));
2015 }
2016
2017 #[test]
2018 fn invocation_recognizer_reports_malformed_discriminators_instead_of_absence() {
2019 for value in [
2020 json!({"type": "1", "target": "GatewayTrade", "arguments": []}),
2021 json!({"type": 1.5}),
2022 json!({"type": -1}),
2023 json!({"type": null}),
2024 json!({"type": [1]}),
2025 json!({}),
2026 json!({"target": "GatewayTrade"}),
2027 json!([1, 2]),
2028 json!("frame"),
2029 ] {
2030 assert!(
2031 matches!(
2032 SignalRInvocation::from_value(&value),
2033 Err(RealtimeError::Protocol(_))
2034 ),
2035 "malformed discriminator {value} must be a typed protocol error"
2036 );
2037 }
2038 for value in [
2039 json!({"type": 6}),
2040 json!({"type": 3, "invocationId": "1"}),
2041 json!({"type": 7, "allowReconnect": false}),
2042 ] {
2043 assert!(
2044 matches!(SignalRInvocation::from_value(&value), Ok(None)),
2045 "another unsigned message type {value} must remain a recognition miss"
2046 );
2047 }
2048 }
2049
2050 #[test]
2051 fn invocation_rejects_malformed_market_arguments() {
2052 for value in [
2053 json!({
2054 "type": 1,
2055 "target": "GatewayTrade",
2056 "arguments": [42, {"price": 1.25}],
2057 }),
2058 json!({
2059 "type": 1,
2060 "target": "GatewayTrade",
2061 "arguments": ["CON.F.US.MNQ.M26", {"price": 1.25}, "extra"],
2062 }),
2063 ] {
2064 assert!(matches!(
2065 SignalRInvocation::from_value(&value),
2066 Err(RealtimeError::Protocol(_))
2067 ));
2068 }
2069 }
2070
2071 #[test]
2072 fn handshake_accepts_coalesced_tail() {
2073 let tail = validate_handshake("{}\u{001e}{\"type\":6}\u{001e}")
2074 .unwrap_or_else(|error| panic!("fixture handshake must decode: {error}"));
2075 assert_eq!(tail.as_deref(), Some("{\"type\":6}\u{001e}"));
2076 }
2077
2078 #[test]
2079 fn handshake_rejects_provider_error_without_retaining_detail() {
2080 let result = validate_handshake("{\"error\":\"secret provider detail\"}\u{001e}");
2081 assert!(matches!(result, Err(RealtimeError::Handshake(_))));
2082 assert!(!format!("{:?}", result.err()).contains("secret provider detail"));
2083 }
2084
2085 #[test]
2086 fn completion_requires_an_unambiguous_protocol_shape() {
2087 for value in [
2088 json!({"type": 3}),
2089 json!({"type": 3, "invocationId": 1}),
2090 json!({"type": 3, "invocationId": "1", "error": false}),
2091 json!({"type": 3, "invocationId": "1", "error": "rejected", "result": null}),
2092 ] {
2093 assert!(matches!(
2094 completion(&value),
2095 Err(RealtimeError::Protocol(_))
2096 ));
2097 }
2098
2099 assert_eq!(
2100 completion(&json!({"type": 3, "invocationId": "1"}))
2101 .unwrap_or_else(|error| panic!("void completion must decode: {error}")),
2102 ("1", Ok(()))
2103 );
2104 assert_eq!(
2105 completion(&json!({"type": 3, "invocationId": "1", "result": 42}))
2106 .unwrap_or_else(|error| panic!("result completion must decode: {error}")),
2107 ("1", Ok(()))
2108 );
2109 assert_eq!(
2110 completion(&json!({"type": 3, "invocationId": "1", "error": "rejected"}))
2111 .unwrap_or_else(|error| panic!("error completion must decode: {error}")),
2112 ("1", Err(()))
2113 );
2114 }
2115
2116 #[test]
2117 fn malformed_transport_frames_are_not_published_as_messages() {
2118 let credentials = crate::Credentials::new("user", "key")
2119 .unwrap_or_else(|error| panic!("fixture credentials must be valid: {error}"));
2120 let client = crate::Client::builder(credentials)
2121 .build()
2122 .unwrap_or_else(|error| panic!("fixture client must build: {error}"));
2123 let realtime = client.realtime(Hub::Market);
2124 let mut events = realtime
2125 .take_event_receiver()
2126 .unwrap_or_else(|| panic!("event receiver must be available"));
2127
2128 for frame in [
2129 "{}\u{001e}",
2130 "{\"type\":\"1\"}\u{001e}",
2131 "{\"type\":3,\"invocationId\":1}\u{001e}",
2132 "{\"type\":3,\"invocationId\":\"1\",\"error\":null}\u{001e}",
2133 "{\"type\":3,\"invocationId\":\"1\",\"error\":\"x\",\"result\":null}\u{001e}",
2134 "{\"type\":7,\"error\":null}\u{001e}",
2135 ] {
2136 assert!(matches!(
2137 realtime
2138 .inner
2139 .process_record(1, frame.trim_end_matches(SIGNALR_TERMINATOR)),
2140 Err(RealtimeError::Protocol(_))
2141 ));
2142 }
2143 assert!(matches!(
2144 events.events.try_recv(),
2145 Err(mpsc::error::TryRecvError::Empty)
2146 ));
2147 }
2148
2149 #[tokio::test]
2150 async fn malformed_record_framing_is_rejected_before_publication() {
2151 let realtime = fixture_realtime();
2152 let mut events = realtime
2153 .take_event_receiver()
2154 .unwrap_or_else(|| panic!("event receiver must be available"));
2155 install_connected_generation(&realtime.inner, 1);
2156
2157 for batch in [
2158 "{\"type\":1}",
2159 "{\"type\":1}\u{001e}\u{001e}{\"type\":1}\u{001e}",
2160 "\u{001e}{\"type\":1}\u{001e}",
2161 "\u{001e}",
2162 ] {
2163 assert!(matches!(
2164 realtime.inner.process_text(1, batch).await,
2165 Ok(ProcessOutcome::Continue)
2166 ));
2167 }
2168 assert!(matches!(
2169 events.events.try_recv(),
2170 Err(mpsc::error::TryRecvError::Empty)
2171 ));
2172 }
2173
2174 #[test]
2175 fn connect_claim_rechecks_the_gap_after_waiting_for_lifecycle() {
2176 let realtime = fixture_realtime();
2177 let before_lock = Arc::new(std::sync::Barrier::new(2));
2178 let release_lock = Arc::new(std::sync::Barrier::new(2));
2179 let worker_inner = Arc::clone(&realtime.inner);
2180 let worker_before = Arc::clone(&before_lock);
2181 let worker_release = Arc::clone(&release_lock);
2182 let claim = std::thread::spawn(move || {
2183 worker_inner.begin_connect_with_pre_lock(false, || {
2184 worker_before.wait();
2185 worker_release.wait();
2186 })
2187 });
2188
2189 before_lock.wait();
2190 realtime
2191 .inner
2192 .event_flow
2193 .start_generation(1)
2194 .unwrap_or_else(|error| panic!("fixture generation must start: {error}"));
2195 realtime.inner.event_flow.mark_gap(1);
2196 realtime.inner.event_flow.finish_generation(1);
2197 release_lock.wait();
2198
2199 let result = claim
2200 .join()
2201 .unwrap_or_else(|payload| std::panic::resume_unwind(payload));
2202 assert!(matches!(result, Err(RealtimeError::TransportGapPending)));
2203 assert!(matches!(
2204 *realtime.inner.lifecycle.lock(),
2205 Lifecycle::Disconnected
2206 ));
2207 }
2208
2209 #[tokio::test(start_paused = true)]
2210 async fn disconnect_waiter_timeout_keeps_the_owner_session_fenced() {
2211 let realtime = fixture_realtime();
2212 let _writer_rx = install_paused_connected_generation(&realtime.inner, 1);
2213 let DisconnectAction::Close(owner_session) = realtime.inner.begin_disconnect() else {
2214 panic!("fixture owner must claim the connected session");
2215 };
2216 let waiter_client = realtime.clone();
2217 let waiter = tokio::spawn(async move { waiter_client.disconnect().await });
2218 tokio::task::yield_now().await;
2219
2220 tokio::time::advance(CLOSE_TIMEOUT).await;
2221 let result = waiter
2222 .await
2223 .unwrap_or_else(|error| panic!("waiter must join: {error}"));
2224 assert!(matches!(result, Err(RealtimeError::Close)));
2225 assert!(matches!(
2226 *realtime.inner.lifecycle.lock(),
2227 Lifecycle::Closing {
2228 generation: 1,
2229 was_connected: true,
2230 }
2231 ));
2232 assert!(matches!(
2233 realtime.connect().await,
2234 Err(RealtimeError::AlreadyConnected)
2235 ));
2236
2237 drop(owner_session);
2238 realtime.inner.finish_closing(1);
2239 assert!(matches!(
2240 *realtime.inner.lifecycle.lock(),
2241 Lifecycle::Disconnected
2242 ));
2243 }
2244
2245 #[tokio::test(start_paused = true)]
2246 async fn disconnect_waiter_timeout_keeps_the_connect_claim_fenced() {
2247 let realtime = fixture_realtime();
2248 let cancellation = realtime.inner.owner_cancel.child_token();
2249 *realtime.inner.lifecycle.lock() = Lifecycle::Connecting {
2250 generation: 1,
2251 cancellation: cancellation.clone(),
2252 };
2253 realtime.inner.generation.store(1, Ordering::Release);
2254 let owner_claim = ConnectClaim {
2255 inner: Arc::downgrade(&realtime.inner),
2256 generation: 1,
2257 cancellation,
2258 complete: false,
2259 };
2260 assert!(matches!(
2261 realtime.inner.begin_disconnect(),
2262 DisconnectAction::Wait(1)
2263 ));
2264 let waiter_client = realtime.clone();
2265 let waiter = tokio::spawn(async move { waiter_client.disconnect().await });
2266 tokio::task::yield_now().await;
2267
2268 tokio::time::advance(CLOSE_TIMEOUT).await;
2269 let result = waiter
2270 .await
2271 .unwrap_or_else(|error| panic!("waiter must join: {error}"));
2272 assert!(matches!(result, Err(RealtimeError::Close)));
2273 assert!(matches!(
2274 *realtime.inner.lifecycle.lock(),
2275 Lifecycle::Closing {
2276 generation: 1,
2277 was_connected: false,
2278 }
2279 ));
2280 assert!(matches!(
2281 realtime.connect().await,
2282 Err(RealtimeError::AlreadyConnected)
2283 ));
2284
2285 drop(owner_claim);
2286 assert!(matches!(
2287 *realtime.inner.lifecycle.lock(),
2288 Lifecycle::Disconnected
2289 ));
2290 }
2291
2292 #[tokio::test]
2293 async fn ended_boundary_waits_for_both_socket_producers_to_drop() {
2294 struct Probe(Arc<AtomicUsize>);
2295 impl Drop for Probe {
2296 fn drop(&mut self) {
2297 self.0.fetch_add(1, Ordering::SeqCst);
2298 }
2299 }
2300 let realtime = fixture_realtime();
2301 let stopped = Arc::new(AtomicUsize::new(0));
2302 let (release, wait) = oneshot::channel::<()>();
2303 let (writer, _writes) = mpsc::channel(1);
2304 let cancel = CancellationToken::new();
2305 let reader_probe = Probe(Arc::clone(&stopped));
2306 let writer_probe = Probe(Arc::clone(&stopped));
2307 let writer_cancel = cancel.clone();
2308 let session = Session {
2309 runtime: tokio::runtime::Handle::current(),
2310 generation: 1,
2311 writer,
2312 cancellation: cancel,
2313 reader_task: tokio::spawn(async move {
2314 let _probe = reader_probe;
2315 let _ = wait.await;
2316 Ok(())
2317 }),
2318 writer_task: tokio::spawn(async move {
2319 let _probe = writer_probe;
2320 writer_cancel.cancelled().await;
2321 Ok(())
2322 }),
2323 };
2324 realtime
2325 .inner
2326 .event_flow
2327 .start_generation(1)
2328 .unwrap_or_else(|e| panic!("start: {e}"));
2329 *realtime.inner.lifecycle.lock() = Lifecycle::Connected(session);
2330 let mut events = realtime
2331 .take_event_receiver()
2332 .unwrap_or_else(|| panic!("receiver"));
2333 for sequence in 0..3 {
2334 assert!(matches!(
2335 realtime.inner.publish(
2336 1,
2337 RealtimeEvent::Message(json!({"sequence": sequence})),
2338 EVENT_BASE_WEIGHT,
2339 ),
2340 Ok(PublishOutcome::Published)
2341 ));
2342 }
2343 realtime.inner.end_generation(1);
2344 for sequence in 0..3 {
2345 assert_eq!(
2346 events.recv().await,
2347 Some(RealtimeEvent::Message(json!({"sequence": sequence})))
2348 );
2349 }
2350 assert!(
2351 tokio::time::timeout(Duration::from_millis(10), events.recv_message())
2352 .await
2353 .is_err()
2354 );
2355 assert!(stopped.load(Ordering::SeqCst) < 2);
2356 assert!(release.send(()).is_ok());
2357 let ended = events
2358 .recv_message()
2359 .await
2360 .unwrap_or_else(|| panic!("ended boundary"));
2361 assert_eq!(ended.generation, RealtimeGeneration(1));
2362 assert_eq!(ended.event, RealtimeEvent::Disconnected);
2363 assert_eq!(stopped.load(Ordering::SeqCst), 2);
2364 }
2365
2366 #[tokio::test]
2367 async fn gap_follows_accepted_events_without_waiting_for_generation_end() {
2368 let flow = EventFlow::new();
2369 let (events_tx, events_rx) = mpsc::unbounded_channel();
2370 let mut receiver = RealtimeEventReceiver {
2371 events: events_rx,
2372 flow: Arc::clone(&flow),
2373 gap_reported: false,
2374 };
2375 let generation = 7;
2376 flow.start_generation(generation)
2377 .unwrap_or_else(|error| panic!("generation must start: {error}"));
2378 flow.publish(
2379 &events_tx,
2380 generation,
2381 RealtimeEvent::Message(json!({"sequence": 1})),
2382 EVENT_BASE_WEIGHT,
2383 )
2384 .unwrap_or_else(|error| panic!("first event must enter the queue: {error}"));
2385
2386 let (terminal_staged_tx, terminal_staged_rx) = oneshot::channel();
2387 let (finish_tx, finish_rx) = oneshot::channel();
2388 let producer_flow = Arc::clone(&flow);
2389 let producer = tokio::spawn(async move {
2390 producer_flow.mark_gap(generation);
2391 producer_flow
2392 .publish(
2393 &events_tx,
2394 generation,
2395 RealtimeEvent::Disconnected,
2396 EVENT_BASE_WEIGHT,
2397 )
2398 .unwrap_or_else(|error| panic!("terminal event must be staged: {error}"));
2399 terminal_staged_tx
2400 .send(())
2401 .unwrap_or_else(|()| panic!("terminal-staged signal must send"));
2402 finish_rx
2403 .await
2404 .unwrap_or_else(|error| panic!("finish signal must arrive: {error}"));
2405 producer_flow.finish_generation(generation);
2406 });
2407
2408 assert_eq!(
2409 receiver.recv().await,
2410 Some(RealtimeEvent::Message(json!({"sequence": 1})))
2411 );
2412 terminal_staged_rx
2413 .await
2414 .unwrap_or_else(|error| panic!("terminal-staged signal must arrive: {error}"));
2415 assert_eq!(receiver.recv().await, Some(RealtimeEvent::TransportGap));
2416 receiver.acknowledge_transport_gap();
2417 finish_tx
2418 .send(())
2419 .unwrap_or_else(|()| panic!("finish signal must send"));
2420 producer
2421 .await
2422 .unwrap_or_else(|error| panic!("producer must join: {error}"));
2423
2424 assert_eq!(receiver.recv().await, Some(RealtimeEvent::Disconnected));
2425 flow.start_generation(generation + 1)
2426 .unwrap_or_else(|error| {
2427 panic!("acknowledgement must allow the next generation: {error}")
2428 });
2429 assert_eq!(receiver.recv().await, None);
2430 assert_eq!(flow.queued_weight.load(Ordering::Acquire), 0);
2431 }
2432
2433 #[tokio::test]
2434 async fn stale_generation_cannot_publish_or_latch_a_gap_after_replacement() {
2435 let flow = EventFlow::new();
2436 let (events_tx, events_rx) = mpsc::unbounded_channel();
2437 let mut receiver = RealtimeEventReceiver {
2438 events: events_rx,
2439 flow: Arc::clone(&flow),
2440 gap_reported: false,
2441 };
2442
2443 flow.start_generation(1)
2444 .unwrap_or_else(|error| panic!("first generation must start: {error}"));
2445 assert!(matches!(
2446 flow.start_generation(2),
2447 Err(RealtimeError::ConnectionCancelled)
2448 ));
2449 assert_eq!(
2450 flow.publish(
2451 &events_tx,
2452 1,
2453 RealtimeEvent::Disconnected,
2454 EVENT_BASE_WEIGHT,
2455 )
2456 .unwrap_or_else(|error| panic!("disconnect must publish: {error}")),
2457 PublishOutcome::Published
2458 );
2459 flow.finish_generation(1);
2460 flow.start_generation(2)
2461 .unwrap_or_else(|error| panic!("replacement generation must start: {error}"));
2462 assert_eq!(
2463 flow.publish(&events_tx, 2, RealtimeEvent::Reconnected, EVENT_BASE_WEIGHT,)
2464 .unwrap_or_else(|error| panic!("reconnect must publish: {error}")),
2465 PublishOutcome::Published
2466 );
2467
2468 for weight in [EVENT_BASE_WEIGHT, usize::MAX] {
2469 assert_eq!(
2470 flow.publish(
2471 &events_tx,
2472 1,
2473 RealtimeEvent::Message(json!({"generation": 1})),
2474 weight,
2475 )
2476 .unwrap_or_else(|error| panic!("stale publication must be ignored: {error}")),
2477 PublishOutcome::StaleGeneration
2478 );
2479 }
2480 assert!(!flow.has_unacknowledged_gap());
2481 assert_eq!(
2482 flow.publish(
2483 &events_tx,
2484 2,
2485 RealtimeEvent::Message(json!({"generation": 2})),
2486 EVENT_BASE_WEIGHT,
2487 )
2488 .unwrap_or_else(|error| panic!("replacement message must publish: {error}")),
2489 PublishOutcome::Published
2490 );
2491 flow.finish_generation(2);
2492 drop(events_tx);
2493
2494 assert_eq!(receiver.recv().await, Some(RealtimeEvent::Disconnected));
2495 assert_eq!(receiver.recv().await, Some(RealtimeEvent::Reconnected));
2496 assert_eq!(
2497 receiver.recv().await,
2498 Some(RealtimeEvent::Message(json!({"generation": 2})))
2499 );
2500 assert_eq!(receiver.recv().await, None);
2501 }
2502
2503 #[tokio::test]
2504 async fn stalled_receiver_retains_more_than_the_former_event_limit_across_generations() {
2505 const BURST: usize = 65_537;
2506 let flow = EventFlow::new();
2507 let (events_tx, events_rx) = mpsc::unbounded_channel();
2508 let mut receiver = RealtimeEventReceiver {
2509 events: events_rx,
2510 flow: Arc::clone(&flow),
2511 gap_reported: false,
2512 };
2513 flow.start_generation(1)
2514 .unwrap_or_else(|error| panic!("first generation must start: {error}"));
2515 for sequence in 0..BURST {
2516 let kind = if sequence % 2 == 0 { "market" } else { "order" };
2517 assert!(matches!(
2518 flow.publish(
2519 &events_tx,
2520 1,
2521 RealtimeEvent::Message(json!({"kind": kind, "sequence": sequence})),
2522 EVENT_BASE_WEIGHT,
2523 ),
2524 Ok(PublishOutcome::Published)
2525 ));
2526 }
2527 assert_eq!(receiver.queued_event_count(), BURST);
2528 assert!(receiver.oldest_event_age().is_some());
2529 assert!(!flow.has_unacknowledged_gap());
2530
2531 assert!(matches!(
2532 flow.publish(&events_tx, 1, RealtimeEvent::Disconnected, 0),
2533 Ok(PublishOutcome::Published)
2534 ));
2535 flow.finish_generation(1);
2536 flow.start_generation(2)
2537 .unwrap_or_else(|error| panic!("replacement generation must start: {error}"));
2538 assert!(matches!(
2539 flow.publish(&events_tx, 2, RealtimeEvent::Reconnected, 0),
2540 Ok(PublishOutcome::Published)
2541 ));
2542 flow.finish_generation(2);
2543 drop(events_tx);
2544
2545 for sequence in 0..BURST {
2546 let message = receiver
2547 .recv_message()
2548 .await
2549 .unwrap_or_else(|| panic!("event {sequence} must survive"));
2550 assert_eq!(message.generation, RealtimeGeneration(1));
2551 let RealtimeEvent::Message(value) = message.event else {
2552 panic!("data event {sequence} must survive");
2553 };
2554 assert_eq!(value["sequence"], sequence);
2555 assert_eq!(
2556 value["kind"],
2557 if sequence % 2 == 0 { "market" } else { "order" }
2558 );
2559 }
2560 assert_eq!(
2561 receiver.recv_message().await,
2562 Some(RealtimeMessage {
2563 generation: RealtimeGeneration(1),
2564 event: RealtimeEvent::Disconnected,
2565 })
2566 );
2567 assert_eq!(
2568 receiver.recv_message().await,
2569 Some(RealtimeMessage {
2570 generation: RealtimeGeneration(2),
2571 event: RealtimeEvent::Reconnected,
2572 })
2573 );
2574 assert_eq!(receiver.queued_event_count(), 0);
2575 assert_eq!(receiver.oldest_event_age(), None);
2576 assert_eq!(receiver.recv_message().await, None);
2577 }
2578
2579 #[tokio::test]
2580 async fn outbound_saturation_does_not_discard_inbound_events() {
2581 let (writer, _writer_rx) = mpsc::channel(1);
2582 assert!(send_queued(&writer, Message::Ping(Vec::new().into())).is_ok());
2583 assert!(matches!(
2584 send_queued(&writer, Message::Ping(Vec::new().into())),
2585 Err(RealtimeError::SendQueueFull)
2586 ));
2587
2588 let flow = EventFlow::new();
2589 let (events_tx, events_rx) = mpsc::unbounded_channel();
2590 let mut receiver = RealtimeEventReceiver {
2591 events: events_rx,
2592 flow: Arc::clone(&flow),
2593 gap_reported: false,
2594 };
2595 flow.start_generation(1)
2596 .unwrap_or_else(|error| panic!("generation must start: {error}"));
2597 assert!(matches!(
2598 flow.publish(
2599 &events_tx,
2600 1,
2601 RealtimeEvent::Message(json!({"order": "terminal"})),
2602 EVENT_BASE_WEIGHT,
2603 ),
2604 Ok(PublishOutcome::Published)
2605 ));
2606 assert_eq!(
2607 receiver.recv().await,
2608 Some(RealtimeEvent::Message(json!({"order": "terminal"})))
2609 );
2610 assert!(!flow.has_unacknowledged_gap());
2611 }
2612
2613 #[test]
2614 fn dropping_event_receiver_refuses_new_data() {
2615 let flow = EventFlow::new();
2616 let (events_tx, events_rx) = mpsc::unbounded_channel();
2617 let receiver = RealtimeEventReceiver {
2618 events: events_rx,
2619 flow: Arc::clone(&flow),
2620 gap_reported: false,
2621 };
2622 flow.start_generation(1)
2623 .unwrap_or_else(|error| panic!("generation must start: {error}"));
2624 assert!(matches!(
2625 flow.publish(&events_tx, 1, RealtimeEvent::Connected, 0),
2626 Ok(PublishOutcome::Published)
2627 ));
2628 assert_eq!(receiver.queued_event_count(), 1);
2629 drop(receiver);
2630 assert!(matches!(
2631 flow.publish(&events_tx, 1, RealtimeEvent::Disconnected, 0),
2632 Err(RealtimeError::EventReceiverClosed)
2633 ));
2634 }
2635
2636 #[tokio::test]
2637 async fn stale_close_frame_cannot_disable_the_replacement_watchdog() {
2638 let realtime = fixture_realtime();
2639 install_connected_generation(&realtime.inner, 2);
2640 realtime
2641 .inner
2642 .reconnect_enabled
2643 .store(true, Ordering::Release);
2644 realtime.inner.start_watchdog();
2645 assert!(
2646 realtime
2647 .inner
2648 .watchdog_task
2649 .lock()
2650 .as_ref()
2651 .is_some_and(|task| !task.is_finished())
2652 );
2653
2654 assert_eq!(
2655 realtime
2656 .inner
2657 .process_text(1, "{\"type\":7,\"allowReconnect\":false}\u{001e}")
2658 .await
2659 .unwrap_or_else(|error| panic!("close frame must decode: {error}")),
2660 ProcessOutcome::Close
2661 );
2662 assert!(realtime.inner.reconnect_enabled.load(Ordering::Acquire));
2663 assert!(
2664 realtime
2665 .inner
2666 .watchdog_task
2667 .lock()
2668 .as_ref()
2669 .is_some_and(|task| !task.is_finished())
2670 );
2671
2672 assert_eq!(
2673 realtime
2674 .inner
2675 .process_text(2, "{\"type\":7,\"allowReconnect\":false}\u{001e}")
2676 .await
2677 .unwrap_or_else(|error| panic!("close frame must decode: {error}")),
2678 ProcessOutcome::Close
2679 );
2680 assert!(realtime.inner.reconnect_enabled.load(Ordering::Acquire));
2681 assert!(realtime.inner.watchdog_task.lock().is_some());
2682 }
2683
2684 #[tokio::test]
2685 async fn stale_generation_cannot_refresh_replacement_liveness() {
2686 let realtime = fixture_realtime();
2687 install_connected_generation(&realtime.inner, 2);
2688 let original = Instant::now() - Duration::from_secs(10);
2689 *realtime.inner.last_activity.lock() = original;
2690
2691 realtime.inner.record_activity(1);
2692 assert_eq!(*realtime.inner.last_activity.lock(), original);
2693
2694 realtime.inner.record_activity(2);
2695 assert!(*realtime.inner.last_activity.lock() > original);
2696 }
2697
2698 #[tokio::test]
2699 async fn typed_subscriptions_reject_the_wrong_hub_before_network_io() {
2700 let credentials = crate::Credentials::new("user", "key")
2701 .unwrap_or_else(|error| panic!("fixture credentials must be valid: {error}"));
2702 let client = crate::Client::builder(credentials)
2703 .build()
2704 .unwrap_or_else(|error| panic!("fixture client must build: {error}"));
2705 let market = client.realtime(Hub::Market);
2706 let user = client.realtime(Hub::User);
2707 let contract = ContractId::new("CON.F.US.MNQ.M26")
2708 .unwrap_or_else(|error| panic!("fixture contract must be valid: {error}"));
2709
2710 assert!(matches!(
2711 market.subscribe_accounts().await,
2712 Err(RealtimeError::WrongHub)
2713 ));
2714 assert!(matches!(
2715 user.subscribe_contract_trades(&contract).await,
2716 Err(RealtimeError::WrongHub)
2717 ));
2718 }
2719
2720 #[tokio::test]
2721 async fn transport_gap_fences_connect_until_acknowledged() {
2722 let credentials = crate::Credentials::new("user", "key")
2723 .unwrap_or_else(|error| panic!("fixture credentials must be valid: {error}"));
2724 let client = crate::Client::builder(credentials)
2725 .build()
2726 .unwrap_or_else(|error| panic!("fixture client must build: {error}"));
2727 let realtime = client.realtime(Hub::Market);
2728 let mut events = realtime
2729 .take_event_receiver()
2730 .unwrap_or_else(|| panic!("event receiver must be available"));
2731 let generation = 1;
2732 realtime
2733 .inner
2734 .event_flow
2735 .start_generation(generation)
2736 .unwrap_or_else(|error| panic!("generation must start: {error}"));
2737 realtime.inner.event_flow.mark_gap(generation);
2738 realtime
2739 .inner
2740 .event_flow
2741 .publish(
2742 &realtime.inner.event_tx,
2743 generation,
2744 RealtimeEvent::Disconnected,
2745 EVENT_BASE_WEIGHT,
2746 )
2747 .unwrap_or_else(|error| panic!("terminal event must be staged: {error}"));
2748 realtime.inner.event_flow.finish_generation(generation);
2749
2750 assert!(matches!(
2751 realtime.connect().await,
2752 Err(RealtimeError::TransportGapPending)
2753 ));
2754 assert!(matches!(
2755 events.recv().await,
2756 Some(RealtimeEvent::TransportGap)
2757 ));
2758 assert!(matches!(
2759 events.recv().await,
2760 Some(RealtimeEvent::Disconnected)
2761 ));
2762 assert!(matches!(
2763 realtime.connect().await,
2764 Err(RealtimeError::TransportGapPending)
2765 ));
2766
2767 events.acknowledge_transport_gap();
2768 assert!(matches!(
2769 realtime.connect().await,
2770 Err(RealtimeError::MissingAuthToken)
2771 ));
2772 }
2773
2774 #[tokio::test]
2775 async fn disabled_reconnect_does_not_spawn_a_watchdog() {
2776 let credentials = crate::Credentials::new("user", "key")
2777 .unwrap_or_else(|error| panic!("fixture credentials must be valid: {error}"));
2778 let client = crate::Client::builder(credentials)
2779 .build()
2780 .unwrap_or_else(|error| panic!("fixture client must build: {error}"));
2781 let realtime = client.realtime(Hub::Market);
2782
2783 realtime.inner.start_watchdog();
2784
2785 assert!(realtime.inner.watchdog_task.lock().is_none());
2786 }
2787
2788 #[test]
2789 fn websocket_configuration_accepts_large_provider_batches() {
2790 let config = websocket_config();
2791 assert_eq!(config.read_buffer_size, WEBSOCKET_READ_BUFFER_SIZE);
2792 assert_eq!(config.write_buffer_size, WEBSOCKET_WRITE_BUFFER_SIZE);
2793 assert_eq!(config.max_message_size, None);
2794 assert_eq!(config.max_frame_size, None);
2795 assert!(config.max_write_buffer_size > config.write_buffer_size);
2796 }
2797
2798 #[test]
2799 fn websocket_accept_matches_the_rfc_6455_vector() {
2800 assert_eq!(
2801 websocket_accept("dGhlIHNhbXBsZSBub25jZQ=="),
2802 "s3pPLMBiTxaQ9kYGzzhZRbK+xOo="
2803 );
2804 let key = websocket_key()
2805 .unwrap_or_else(|error| panic!("fixture key generation must succeed: {error}"));
2806 let decoded = BASE64
2807 .decode(key.as_bytes())
2808 .unwrap_or_else(|error| panic!("generated key must be base64: {error}"));
2809 assert_eq!(decoded.len(), 16);
2810 }
2811
2812 #[test]
2813 fn websocket_upgrade_validation_rejects_untrusted_responses() {
2814 let expected_accept = websocket_accept("dGhlIHNhbXBsZSBub25jZQ==");
2815 let mut valid = header::HeaderMap::new();
2816 valid.insert(
2817 header::CONNECTION,
2818 header::HeaderValue::from_static("keep-alive, Upgrade"),
2819 );
2820 valid.insert(
2821 header::UPGRADE,
2822 header::HeaderValue::from_static("WebSocket"),
2823 );
2824 valid.insert(
2825 header::SEC_WEBSOCKET_ACCEPT,
2826 header::HeaderValue::from_str(&expected_accept)
2827 .unwrap_or_else(|error| panic!("fixture accept header must be valid: {error}")),
2828 );
2829
2830 assert!(
2831 validate_websocket_upgrade(
2832 StatusCode::SWITCHING_PROTOCOLS,
2833 Version::HTTP_11,
2834 &valid,
2835 &expected_accept,
2836 )
2837 .is_ok()
2838 );
2839
2840 let mut cases = Vec::new();
2841 cases.push((StatusCode::OK, Version::HTTP_11, valid.clone()));
2842 cases.push((
2843 StatusCode::SWITCHING_PROTOCOLS,
2844 Version::HTTP_2,
2845 valid.clone(),
2846 ));
2847 for missing in [
2848 header::CONNECTION,
2849 header::UPGRADE,
2850 header::SEC_WEBSOCKET_ACCEPT,
2851 ] {
2852 let mut headers = valid.clone();
2853 headers.remove(missing);
2854 cases.push((StatusCode::SWITCHING_PROTOCOLS, Version::HTTP_11, headers));
2855 }
2856 for unsolicited in [
2857 header::SEC_WEBSOCKET_EXTENSIONS,
2858 header::SEC_WEBSOCKET_PROTOCOL,
2859 ] {
2860 let mut headers = valid.clone();
2861 headers.insert(unsolicited, header::HeaderValue::from_static("unsupported"));
2862 cases.push((StatusCode::SWITCHING_PROTOCOLS, Version::HTTP_11, headers));
2863 }
2864 let mut duplicate_accept = valid.clone();
2865 duplicate_accept.append(
2866 header::SEC_WEBSOCKET_ACCEPT,
2867 header::HeaderValue::from_str(&expected_accept)
2868 .unwrap_or_else(|error| panic!("fixture accept header must be valid: {error}")),
2869 );
2870 cases.push((
2871 StatusCode::SWITCHING_PROTOCOLS,
2872 Version::HTTP_11,
2873 duplicate_accept,
2874 ));
2875
2876 for (status, version, headers) in cases {
2877 assert!(matches!(
2878 validate_websocket_upgrade(status, version, &headers, &expected_accept),
2879 Err(RealtimeError::Transport)
2880 ));
2881 }
2882 }
2883
2884 #[test]
2885 fn invocation_encoder_accepts_small_terminated_messages() {
2886 let message = encode_invocation("1", "SubscribeAccounts", &[])
2887 .unwrap_or_else(|error| panic!("small invocation must encode: {error}"));
2888 let Message::Text(text) = message else {
2889 panic!("invocation must encode as text");
2890 };
2891 assert!(text.ends_with(SIGNALR_TERMINATOR));
2892 }
2893
2894 #[test]
2895 fn invocation_encoder_preserves_large_argument_batches() {
2896 let payload = "x".repeat(2 * 1_024 * 1_024);
2897 let message = encode_invocation("1", "Batch", &[Value::String(payload.clone())])
2898 .unwrap_or_else(|error| panic!("large invocation: {error}"));
2899 let Message::Text(text) = message else {
2900 panic!("text invocation");
2901 };
2902 let value: Value = serde_json::from_str(text.trim_end_matches(SIGNALR_TERMINATOR))
2903 .unwrap_or_else(|error| panic!("decode: {error}"));
2904 assert_eq!(value["arguments"][0], payload);
2905 }
2906 #[test]
2907 fn dropping_ready_client_outside_a_runtime_joins_on_its_original_runtime() {
2908 let runtime = tokio::runtime::Builder::new_current_thread()
2909 .enable_all()
2910 .build()
2911 .unwrap_or_else(|error| panic!("fixture runtime: {error}"));
2912 let (realtime, mut events) = runtime.block_on(async {
2913 let realtime = fixture_realtime();
2914 install_connected_generation(&realtime.inner, 1);
2915 let events = realtime
2916 .take_event_receiver()
2917 .unwrap_or_else(|| panic!("receiver"));
2918 (realtime, events)
2919 });
2920 drop(realtime);
2922 runtime.block_on(async {
2923 tokio::time::pause();
2924 assert_eq!(events.recv().await, Some(RealtimeEvent::Disconnected));
2925 assert_eq!(events.recv().await, None);
2926 });
2927 }
2928}