1use std::collections::{BTreeMap, HashMap, VecDeque};
56use std::future::pending;
57use std::sync::atomic::{AtomicU64, Ordering};
58use std::sync::{Arc, Mutex, PoisonError};
59use std::time::Duration;
60
61use tokio::io::{AsyncRead, AsyncWrite};
62use tokio::sync::{broadcast, mpsc, oneshot};
63use tokio::task::JoinHandle;
64use tokio::time::{Instant, sleep, sleep_until, timeout};
65
66use crate::common::config::{
67 ConfigurationOpenApi, DEFAULT_BACKOFF, DEFAULT_RECONNECT_DELAY, DEFAULT_RETRIES,
68 DEFAULT_TIMEOUT,
69};
70use crate::common::errors::{
71 ConnectorError, ErrorCode, Interruption, ServerError, SessionEndCause, SessionEnded,
72 TransportError,
73};
74use crate::common::ids::AccountId;
75use crate::open_api::messages::{
76 ProtoErrorRes, ProtoHeartbeatEvent, ProtoMessage, ProtoOaAccountAuthReq, ProtoOaAccountAuthRes,
77 ProtoOaApplicationAuthReq, ProtoOaApplicationAuthRes, ProtoOaErrorRes,
78};
79use crate::open_api::payload::{Payload, Request, account_of, is_historical, sessions_ended_by};
80use crate::open_api::transport::{Connect, Framed};
81
82pub const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(10);
84
85pub const DEFAULT_COMMAND_CAPACITY: usize = 64;
87
88pub const DEFAULT_EVENT_CAPACITY: usize = 1024;
90
91pub const MAX_COMMAND_CAPACITY: usize = 65_536;
94
95pub const MAX_EVENT_CAPACITY: usize = 65_536;
98
99pub const REQUEST_LIMIT: u32 = 50;
101
102pub const HISTORICAL_LIMIT: u32 = 5;
104
105pub const BACKOFF_LIMIT: Duration = Duration::from_secs(30);
107
108const LIMIT_WINDOW: Duration = Duration::from_millis(1100);
109
110pub const RECONNECT_DELAY_LIMIT: Duration = Duration::from_secs(60);
112
113const SHORTEST_INTERVAL: Duration = Duration::from_millis(1);
114
115pub const SERVER_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(30);
117
118pub const DEFAULT_INBOUND_TIMEOUT: Duration = Duration::from_secs(75);
122
123const LONGEST_WAIT: Duration = Duration::from_secs(30 * 365 * 86_400);
124
125macro_rules! log_debug {
126 ($($argument:tt)*) => {{
127 #[cfg(feature = "logging")]
128 tracing::debug!($($argument)*);
129 }};
130}
131
132#[derive(Debug, Clone, PartialEq, Eq)]
134#[must_use]
135pub struct ConnectionOptions {
136 timeout: Duration,
137 heartbeat_interval: Duration,
138 command_capacity: usize,
139 event_capacity: usize,
140 reconnect_delay: Duration,
141 reconnect_attempts: Option<u32>,
142 request_limit: u32,
143 historical_limit: u32,
144 retries: u32,
145 backoff: Duration,
146 inbound_timeout: Option<Duration>,
147}
148
149impl Default for ConnectionOptions {
150 fn default() -> Self {
151 Self {
152 timeout: DEFAULT_TIMEOUT,
153 heartbeat_interval: HEARTBEAT_INTERVAL,
154 command_capacity: DEFAULT_COMMAND_CAPACITY,
155 event_capacity: DEFAULT_EVENT_CAPACITY,
156 reconnect_delay: DEFAULT_RECONNECT_DELAY,
157 reconnect_attempts: None,
158 request_limit: REQUEST_LIMIT,
159 historical_limit: HISTORICAL_LIMIT,
160 retries: DEFAULT_RETRIES,
161 backoff: DEFAULT_BACKOFF,
162 inbound_timeout: None,
163 }
164 }
165}
166
167impl From<&ConfigurationOpenApi> for ConnectionOptions {
168 fn from(configuration: &ConfigurationOpenApi) -> Self {
169 Self::default()
170 .timeout(configuration.timeout())
171 .reconnect_delay(configuration.reconnect_delay())
172 .retries(configuration.retries())
173 .backoff(configuration.backoff())
174 }
175}
176
177impl ConnectionOptions {
178 pub fn new() -> Self {
180 Self::default()
181 }
182
183 pub fn timeout(mut self, timeout: Duration) -> Self {
187 self.timeout = timeout.max(SHORTEST_INTERVAL);
188 self
189 }
190
191 pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
195 self.heartbeat_interval = interval.max(SHORTEST_INTERVAL);
196 self
197 }
198
199 pub fn command_capacity(mut self, capacity: usize) -> Self {
203 self.command_capacity = capacity.clamp(1, MAX_COMMAND_CAPACITY);
204 self
205 }
206
207 pub fn event_capacity(mut self, capacity: usize) -> Self {
211 self.event_capacity = capacity.clamp(1, MAX_EVENT_CAPACITY);
212 self
213 }
214
215 pub fn reconnect_delay(mut self, delay: Duration) -> Self {
220 self.reconnect_delay = delay.max(SHORTEST_INTERVAL);
221 self
222 }
223
224 pub const fn reconnect_attempts(mut self, attempts: Option<u32>) -> Self {
228 self.reconnect_attempts = attempts;
229 self
230 }
231
232 pub fn request_limit(mut self, limit: u32) -> Self {
234 self.request_limit = limit.max(1);
235 self
236 }
237
238 pub fn historical_limit(mut self, limit: u32) -> Self {
240 self.historical_limit = limit.max(1);
241 self
242 }
243
244 pub const fn retries(mut self, retries: u32) -> Self {
247 self.retries = retries;
248 self
249 }
250
251 pub fn backoff(mut self, backoff: Duration) -> Self {
256 self.backoff = backoff.max(SHORTEST_INTERVAL);
257 self
258 }
259
260 pub fn inbound_timeout(mut self, period: Duration) -> Self {
274 self.inbound_timeout = Some(period.max(SHORTEST_INTERVAL));
275 self
276 }
277
278 fn silence(&self) -> Duration {
279 self.inbound_timeout.unwrap_or(DEFAULT_INBOUND_TIMEOUT)
280 }
281}
282
283#[derive(Debug, Clone, PartialEq, Eq)]
285#[non_exhaustive]
286pub enum Event {
287 Message(ProtoMessage),
289 Disconnected(TransportError),
291 ReconnectFailed {
293 attempt: u32,
295 reason: TransportError,
297 },
298 Reconnected {
300 attempt: u32,
302 },
303 ReplayFailed {
305 key: ReplayKey,
307 error: ConnectorError,
309 },
310 SessionEnded(SessionEnded),
313}
314
315impl Event {
316 #[must_use]
318 pub fn decode<M: Payload>(&self) -> Option<M> {
319 match self {
320 Self::Message(message) => M::from_message(message).ok(),
321 _ => None,
322 }
323 }
324}
325
326#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
328#[non_exhaustive]
329pub struct Stats {
330 pub requests_sent: u64,
332 pub heartbeats_sent: u64,
334 pub heartbeats_received: u64,
336 pub reconnects: u64,
338 pub throttle_retries: u64,
340}
341
342#[derive(Debug, Default)]
343struct Counters {
344 requests_sent: AtomicU64,
345 heartbeats_sent: AtomicU64,
346 heartbeats_received: AtomicU64,
347 reconnects: AtomicU64,
348 throttle_retries: AtomicU64,
349}
350
351fn bump(counter: &AtomicU64) {
352 counter.fetch_add(1, Ordering::Relaxed);
353}
354
355#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
357#[non_exhaustive]
358pub enum ReplayStage {
359 Application,
361 Account,
363 Subscription,
365}
366
367#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
371pub struct ReplayKey {
372 stage: ReplayStage,
373 id: String,
374}
375
376impl ReplayKey {
377 #[must_use]
379 pub fn new(stage: ReplayStage, id: impl Into<String>) -> Self {
380 Self {
381 stage,
382 id: id.into(),
383 }
384 }
385
386 #[must_use]
388 pub const fn stage(&self) -> ReplayStage {
389 self.stage
390 }
391
392 #[must_use]
394 pub fn id(&self) -> &str {
395 &self.id
396 }
397}
398
399#[derive(Clone, PartialEq, Eq)]
400struct Stored {
401 payload_type: u32,
402 payload: Vec<u8>,
403}
404
405type Replays = BTreeMap<ReplayKey, Stored>;
406
407type Reply = oneshot::Sender<Result<ProtoMessage, ConnectorError>>;
408
409struct Outbound {
410 payload_type: u32,
411 payload: Vec<u8>,
412 historical: bool,
413 once: bool,
414 attempts: u32,
415 reply: Reply,
416}
417
418impl Outbound {
419 fn interrupted(&self, by: Interruption) -> ConnectorError {
420 if self.once {
421 ConnectorError::OutcomeUnknown(by)
422 } else {
423 match by {
424 Interruption::Timeout(limit) => ConnectorError::Timeout(limit),
425 Interruption::Transport(reason) => ConnectorError::Transport(reason),
426 Interruption::Skipped => ConnectorError::OutcomeUnknown(by),
427 }
428 }
429 }
430}
431
432#[derive(Default)]
433struct Ended {
434 application: Option<SessionEndCause>,
435 accounts: BTreeMap<AccountId, SessionEndCause>,
436}
437
438impl Ended {
439 fn refusal(&self, payload_type: u32, payload: &[u8]) -> Option<SessionEnded> {
440 if payload_type == ProtoOaApplicationAuthReq::PAYLOAD_TYPE {
441 return None;
442 }
443 if let Some(cause) = &self.application {
444 return Some(SessionEnded::new(None, cause.clone()));
445 }
446 if payload_type == ProtoOaAccountAuthReq::PAYLOAD_TYPE {
447 return None;
448 }
449 let account = account_of(payload_type, payload)?;
450 let cause = self.accounts.get(&account)?;
451 Some(SessionEnded::new(Some(account), cause.clone()))
452 }
453
454 fn granted(&mut self, answer: &ProtoMessage) {
455 if answer.payload_type == ProtoOaApplicationAuthRes::PAYLOAD_TYPE {
456 self.application = None;
457 } else if answer.payload_type == ProtoOaAccountAuthRes::PAYLOAD_TYPE {
458 let payload = answer.payload.as_deref().unwrap_or_default();
459 if let Some(account) = account_of(answer.payload_type, payload) {
460 self.accounts.remove(&account);
461 }
462 }
463 }
464}
465
466fn token_refused(key: &ReplayKey, stored: &Stored, error: &ConnectorError) -> Option<SessionEnded> {
467 let ConnectorError::Server(error) = error else {
468 return None;
469 };
470 let dead = matches!(
471 error.code,
472 ErrorCode::ChAccessTokenInvalid | ErrorCode::OaAuthTokenExpired
473 );
474 if !dead || key.stage() != ReplayStage::Account {
475 return None;
476 }
477 let account = account_of(stored.payload_type, &stored.payload)?;
478 Some(SessionEnded::new(
479 Some(account),
480 SessionEndCause::TokenRefused(error.code.clone()),
481 ))
482}
483
484enum Command {
485 Request(Outbound),
486 Replay(ReplayKey, Option<Stored>),
487 Reconnect,
488 Shutdown,
489}
490
491#[derive(Debug, Clone)]
495pub struct Connection {
496 commands: mpsc::Sender<Command>,
497 events: broadcast::Sender<Event>,
498 counters: Arc<Counters>,
499 task: Arc<Mutex<Option<JoinHandle<()>>>>,
500}
501
502impl Connection {
503 pub async fn open<C: Connect>(
511 connector: C,
512 host: &str,
513 port: u16,
514 options: ConnectionOptions,
515 ) -> Result<Self, ConnectorError> {
516 let stream = connector.connect(host, port).await?;
517 log_debug!(host, port, "connected");
518 let (commands, receiver) = mpsc::channel(options.command_capacity);
519 let (events, _) = broadcast::channel(options.event_capacity);
520 let counters = Arc::new(Counters::default());
521 let actor = Actor {
522 connector,
523 host: host.to_owned(),
524 port,
525 heartbeat_at: within(Instant::now(), options.heartbeat_interval),
526 silent_at: within(Instant::now(), options.silence()),
527 options,
528 commands: receiver,
529 events: events.clone(),
530 counters: Arc::clone(&counters),
531 queue: VecDeque::new(),
532 pending: HashMap::new(),
533 deadlines: VecDeque::new(),
534 replays: Replays::new(),
535 ended: Ended::default(),
536 throttles: [Throttle::default(), Throttle::default()],
537 paused_until: None,
538 send_at: None,
539 next_id: 0,
540 };
541 let task = tokio::spawn(actor.run(Framed::new(stream)));
542 Ok(Self {
543 commands,
544 events,
545 counters,
546 task: Arc::new(Mutex::new(Some(task))),
547 })
548 }
549
550 pub async fn request<R: Request>(&self, request: &R) -> Result<R::Response, ConnectorError> {
560 let message = self
561 .send(
562 R::PAYLOAD_TYPE,
563 request.encode_to_vec(),
564 R::HISTORICAL,
565 false,
566 )
567 .await?;
568 R::Response::from_message(&message)
569 }
570
571 pub async fn request_message(
577 &self,
578 payload_type: u32,
579 payload: Vec<u8>,
580 ) -> Result<ProtoMessage, ConnectorError> {
581 self.send(payload_type, payload, is_historical(payload_type), false)
582 .await
583 }
584
585 pub async fn request_once(
605 &self,
606 payload_type: u32,
607 payload: Vec<u8>,
608 ) -> Result<ProtoMessage, ConnectorError> {
609 self.send(payload_type, payload, is_historical(payload_type), true)
610 .await
611 }
612
613 async fn send(
614 &self,
615 payload_type: u32,
616 payload: Vec<u8>,
617 historical: bool,
618 once: bool,
619 ) -> Result<ProtoMessage, ConnectorError> {
620 let (reply, response) = oneshot::channel();
621 let outbound = Outbound {
622 payload_type,
623 payload,
624 reply,
625 historical,
626 once,
627 attempts: 0,
628 };
629 self.commands
630 .send(Command::Request(outbound))
631 .await
632 .map_err(|_| ConnectorError::Closed)?;
633 response.await.map_err(|_| ConnectorError::Closed)?
634 }
635
636 pub async fn register_replay<R: Request>(
646 &self,
647 key: ReplayKey,
648 request: &R,
649 ) -> Result<(), ConnectorError> {
650 let stored = Stored {
651 payload_type: R::PAYLOAD_TYPE,
652 payload: request.encode_to_vec(),
653 };
654 self.command(Command::Replay(key, Some(stored))).await
655 }
656
657 pub async fn remove_replay(&self, key: ReplayKey) -> Result<(), ConnectorError> {
669 self.command(Command::Replay(key, None)).await
670 }
671
672 pub async fn force_reconnect(&self) -> Result<(), ConnectorError> {
678 self.command(Command::Reconnect).await
679 }
680
681 async fn command(&self, command: Command) -> Result<(), ConnectorError> {
682 self.commands
683 .send(command)
684 .await
685 .map_err(|_| ConnectorError::Closed)
686 }
687
688 #[must_use]
693 pub fn events(&self) -> broadcast::Receiver<Event> {
694 self.events.subscribe()
695 }
696
697 pub async fn closed(&self) {
700 self.commands.closed().await;
701 }
702
703 #[must_use]
705 pub fn stats(&self) -> Stats {
706 Stats {
707 requests_sent: self.counters.requests_sent.load(Ordering::Relaxed),
708 heartbeats_sent: self.counters.heartbeats_sent.load(Ordering::Relaxed),
709 heartbeats_received: self.counters.heartbeats_received.load(Ordering::Relaxed),
710 reconnects: self.counters.reconnects.load(Ordering::Relaxed),
711 throttle_retries: self.counters.throttle_retries.load(Ordering::Relaxed),
712 }
713 }
714
715 pub async fn shutdown(&self) {
720 let _ = self.commands.send(Command::Shutdown).await;
721 let task = self
722 .task
723 .lock()
724 .unwrap_or_else(PoisonError::into_inner)
725 .take();
726 if let Some(task) = task {
727 let _ = task.await;
728 }
729 }
730}
731
732#[derive(Debug)]
733enum Exit {
734 Shutdown,
735 Lost(TransportError),
736}
737
738struct Actor<C: Connect> {
739 connector: C,
740 host: String,
741 port: u16,
742 options: ConnectionOptions,
743 commands: mpsc::Receiver<Command>,
744 events: broadcast::Sender<Event>,
745 counters: Arc<Counters>,
746 queue: VecDeque<Outbound>,
747 pending: HashMap<String, Outbound>,
748 deadlines: VecDeque<(Option<Instant>, String)>,
749 replays: Replays,
750 ended: Ended,
751 throttles: [Throttle; 2],
752 paused_until: Option<Instant>,
753 send_at: Option<Instant>,
754 next_id: u64,
755 heartbeat_at: Option<Instant>,
756 silent_at: Option<Instant>,
757}
758
759impl<C: Connect> Actor<C> {
760 async fn run(mut self, mut framed: Framed<C::Stream>) {
761 let error = loop {
762 match self.serve(&mut framed).await {
763 Exit::Shutdown => break ConnectorError::Closed,
764 Exit::Lost(reason) => {
765 log_debug!(%reason, "connection lost");
766 close(&mut framed, self.options.timeout).await;
767 self.fail_in_flight(&reason);
768 let _ = self.events.send(Event::Disconnected(reason.clone()));
769 match self.reconnect(reason).await {
770 Ok(fresh) => framed = fresh,
771 Err(error) => break error,
772 }
773 }
774 }
775 };
776 self.fail_all(&error);
777 close(&mut framed, self.options.timeout).await;
778 }
779
780 async fn reconnect(
781 &mut self,
782 reason: TransportError,
783 ) -> Result<Framed<C::Stream>, ConnectorError> {
784 let mut last = reason;
785 let mut delay = self.options.reconnect_delay;
786 let longest = RECONNECT_DELAY_LIMIT.max(delay);
787 let mut attempt = 0_u32;
788 loop {
789 if self
790 .options
791 .reconnect_attempts
792 .is_some_and(|limit| attempt >= limit)
793 {
794 return Err(ConnectorError::Transport(last));
795 }
796 attempt = attempt.saturating_add(1);
797 hold(&mut self.commands, &mut self.replays, &last, sleep(delay)).await?;
798 delay = delay.saturating_mul(2).min(longest);
799 match self.restore(&last).await? {
800 Ok((framed, replays_sent)) => {
801 log_debug!(attempt, "reconnected");
802 bump(&self.counters.reconnects);
803 let seeded = replays_sent
804 .min(usize::try_from(self.options.request_limit).unwrap_or(usize::MAX));
805 if let Some(throttle) = self.throttles.first_mut() {
806 throttle
807 .sent
808 .extend(std::iter::repeat_n(Instant::now(), seeded));
809 }
810 self.heartbeat_at = within(Instant::now(), self.options.heartbeat_interval);
811 self.silent_at = within(Instant::now(), self.options.silence());
812 let _ = self.events.send(Event::Reconnected { attempt });
813 return Ok(framed);
814 }
815 Err(reason) => {
816 log_debug!(attempt, %reason, "reconnection failed");
817 let _ = self.events.send(Event::ReconnectFailed {
818 attempt,
819 reason: reason.clone(),
820 });
821 last = reason;
822 }
823 }
824 }
825 }
826
827 async fn restore(
828 &mut self,
829 reason: &TransportError,
830 ) -> Result<Result<(Framed<C::Stream>, usize), TransportError>, ConnectorError> {
831 let limit = self.options.timeout;
832 let spacing = LIMIT_WINDOW / self.options.request_limit;
833 let connect = self.connector.connect(&self.host, self.port);
834 let mut framed = match hold(&mut self.commands, &mut self.replays, reason, connect).await? {
835 Ok(stream) => Framed::new(stream),
836 Err(reason) => return Ok(Err(reason)),
837 };
838 let mut sent = Replays::new();
839 loop {
840 let next = self
841 .replays
842 .iter()
843 .find(|(key, stored)| sent.get(*key) != Some(*stored))
844 .map(|(key, stored)| (key.clone(), stored.clone()));
845 let Some((key, stored)) = next else {
846 return Ok(Ok((framed, sent.len())));
847 };
848 sent.insert(key.clone(), stored.clone());
849 let id = self.fresh_id();
850 let mut pushed = Vec::new();
851 let step = Box::pin(replay(
852 &mut framed,
853 id,
854 stored.clone(),
855 limit,
856 &self.events,
857 &mut pushed,
858 ));
859 let answer = match hold(&mut self.commands, &mut self.replays, reason, step).await? {
860 Ok(answer) => answer,
861 Err(reason) => return Ok(Err(reason)),
862 };
863 for session in pushed {
864 self.end(session);
865 }
866 match answer {
867 Ok(answer) => self.ended.granted(&answer),
868 Err(error) => {
869 log_debug!(%error, "replayed request failed");
870 let refused = token_refused(&key, &stored, &error);
871 let _ = self.events.send(Event::ReplayFailed { key, error });
872 if let Some(session) = refused {
873 self.end(session);
874 }
875 }
876 }
877 hold(
878 &mut self.commands,
879 &mut self.replays,
880 reason,
881 sleep(spacing),
882 )
883 .await?;
884 }
885 }
886
887 async fn serve(&mut self, framed: &mut Framed<C::Stream>) -> Exit {
888 loop {
889 if let Err(reason) = self.flush(framed).await {
890 return Exit::Lost(reason);
891 }
892 let wake = self.next_wake();
893 let capacity = self.options.command_capacity;
894 tokio::select! {
895 incoming = framed.receive() => match incoming {
896 Ok(message) => self.heard(message),
897 Err(reason) => return Exit::Lost(reason),
898 },
899 command = self.commands.recv(), if self.queue.len() < capacity => match command {
900 Some(Command::Request(outbound)) => self.queue.push_back(outbound),
901 Some(Command::Replay(key, stored)) => store(&mut self.replays, key, stored),
902 Some(Command::Reconnect) => return Exit::Lost(TransportError::Closed),
903 Some(Command::Shutdown) | None => return Exit::Shutdown,
904 },
905 () = reached(wake) => {
906 if let Err(reason) = self.tick(framed).await {
907 return Exit::Lost(reason);
908 }
909 }
910 }
911 }
912 }
913
914 fn end(&mut self, session: SessionEnded) {
915 log_debug!(%session, "authorisation ended");
916 if let Some(account) = session.account {
917 self.replays.retain(|_, stored| {
918 account_of(stored.payload_type, &stored.payload) != Some(account)
919 });
920 self.ended.accounts.insert(account, session.cause.clone());
921 } else {
922 for stored in self.replays.values() {
923 if let Some(account) = account_of(stored.payload_type, &stored.payload) {
924 self.ended.accounts.insert(account, session.cause.clone());
925 }
926 }
927 self.replays.clear();
928 self.ended.application = Some(session.cause.clone());
929 }
930 let _ = self.events.send(Event::SessionEnded(session));
931 }
932
933 fn fresh_id(&mut self) -> String {
934 self.next_id = self.next_id.saturating_add(1);
935 self.next_id.to_string()
936 }
937
938 fn heard(&mut self, message: ProtoMessage) {
939 self.silent_at = within(Instant::now(), self.options.silence());
940 self.dispatch(message);
941 }
942
943 fn next_wake(&self) -> Option<Instant> {
944 let deadline = self.deadlines.front().and_then(|(deadline, _)| *deadline);
945 [self.heartbeat_at, self.silent_at, deadline, self.send_at]
946 .into_iter()
947 .flatten()
948 .min()
949 }
950
951 fn next_ready(&mut self, now: Instant) -> Option<usize> {
952 self.send_at = None;
953 if let Some(until) = self.paused_until {
954 if until > now {
955 if !self.queue.is_empty() {
956 self.send_at = Some(until);
957 }
958 return None;
959 }
960 self.paused_until = None;
961 }
962 let limits = [self.options.request_limit, self.options.historical_limit];
963 let mut ready = [None, None];
964 for ((throttle, limit), ready) in self.throttles.iter_mut().zip(limits).zip(&mut ready) {
965 *ready = throttle.ready_at(now, limit);
966 }
967 for (index, outbound) in self.queue.iter().enumerate() {
968 match ready
969 .get(usize::from(outbound.historical))
970 .copied()
971 .flatten()
972 {
973 None => return Some(index),
974 Some(at) => self.send_at = Some(self.send_at.map_or(at, |known| known.min(at))),
975 }
976 }
977 None
978 }
979
980 async fn flush(&mut self, framed: &mut Framed<C::Stream>) -> Result<(), TransportError> {
981 loop {
982 let now = Instant::now();
983 let Some(outbound) = self
984 .next_ready(now)
985 .and_then(|index| self.queue.remove(index))
986 else {
987 return Ok(());
988 };
989 if outbound.reply.is_closed() {
990 continue;
991 }
992 if let Some(ended) = self.ended.refusal(outbound.payload_type, &outbound.payload) {
993 let _ = outbound.reply.send(Err(ended.into()));
994 continue;
995 }
996 let id = self.fresh_id();
997 let message = ProtoMessage {
998 payload_type: outbound.payload_type,
999 payload: Some(outbound.payload.clone()),
1000 client_msg_id: Some(id.clone()),
1001 };
1002 match write(framed, &message, self.options.timeout).await {
1003 Ok(()) => {
1004 bump(&self.counters.requests_sent);
1005 if let Some(throttle) = self.throttles.get_mut(usize::from(outbound.historical))
1006 {
1007 throttle.sent.push_back(now);
1008 }
1009 self.heartbeat_at = within(now, self.options.heartbeat_interval);
1010 self.deadlines
1011 .push_back((within(now, self.options.timeout), id.clone()));
1012 self.pending.insert(id, outbound);
1013 }
1014 Err(error @ TransportError::FrameTooLong { .. }) => {
1015 let _ = outbound.reply.send(Err(error.into()));
1016 }
1017 Err(reason) => {
1018 let error = outbound.interrupted(Interruption::Transport(reason.clone()));
1019 let _ = outbound.reply.send(Err(error));
1020 return Err(reason);
1021 }
1022 }
1023 }
1024 }
1025
1026 async fn tick(&mut self, framed: &mut Framed<C::Stream>) -> Result<(), TransportError> {
1027 let now = Instant::now();
1028 while let Some((deadline, id)) = self.deadlines.pop_front() {
1029 if deadline.is_none_or(|deadline| deadline > now) {
1030 self.deadlines.push_front((deadline, id));
1031 break;
1032 }
1033 if let Some(outbound) = self.pending.remove(&id) {
1034 let error = outbound.interrupted(Interruption::Timeout(self.options.timeout));
1035 let _ = outbound.reply.send(Err(error));
1036 }
1037 }
1038 if self.silent_at.is_some_and(|silent_at| now >= silent_at) {
1039 let silence = self.options.silence();
1040 let late = timeout(Duration::ZERO, framed.receive())
1041 .await
1042 .unwrap_or(Err(TransportError::Idle(silence)));
1043 self.heard(late?);
1044 }
1045 if self
1046 .heartbeat_at
1047 .is_some_and(|heartbeat_at| now >= heartbeat_at)
1048 {
1049 let beat = ProtoHeartbeatEvent::default().to_message(None);
1050 write(framed, &beat, self.options.timeout).await?;
1051 bump(&self.counters.heartbeats_sent);
1052 self.heartbeat_at = within(Instant::now(), self.options.heartbeat_interval);
1053 }
1054 Ok(())
1055 }
1056
1057 fn dispatch(&mut self, message: ProtoMessage) {
1058 if message.payload_type == ProtoHeartbeatEvent::PAYLOAD_TYPE {
1059 bump(&self.counters.heartbeats_received);
1060 return;
1061 }
1062 let waiting = message
1063 .client_msg_id
1064 .as_ref()
1065 .and_then(|id| self.pending.remove(id));
1066 let Some(mut outbound) = waiting else {
1067 let sessions = sessions_ended_by(&message);
1068 let _ = self.events.send(Event::Message(message));
1069 for session in sessions {
1070 self.end(session);
1071 }
1072 return;
1073 };
1074 self.ended.granted(&message);
1075 if outbound.once {
1076 let _ = self.events.send(Event::Message(message.clone()));
1077 }
1078 let result = outcome(message);
1079 if !outbound.once && over_limit(&result) && outbound.attempts < self.options.retries {
1080 let pause = self
1081 .options
1082 .backoff
1083 .saturating_mul(1 << outbound.attempts.min(16))
1084 .min(BACKOFF_LIMIT.max(self.options.backoff));
1085 let until = after(Instant::now(), pause);
1086 log_debug!(
1087 payload_type = outbound.payload_type,
1088 ?pause,
1089 "limit exceeded"
1090 );
1091 self.paused_until = Some(self.paused_until.map_or(until, |known| known.max(until)));
1092 outbound.attempts += 1;
1093 bump(&self.counters.throttle_retries);
1094 self.queue.push_front(outbound);
1095 } else {
1096 let _ = outbound.reply.send(result);
1097 }
1098 }
1099
1100 fn fail_in_flight(&mut self, reason: &TransportError) {
1101 self.deadlines.clear();
1102 for (_, outbound) in self.pending.drain() {
1103 let error = outbound.interrupted(Interruption::Transport(reason.clone()));
1104 let _ = outbound.reply.send(Err(error));
1105 }
1106 for outbound in self.queue.drain(..) {
1107 let _ = outbound.reply.send(Err(reason.clone().into()));
1108 }
1109 self.reset_pacing();
1110 }
1111
1112 fn close_in_flight(&mut self, error: &ConnectorError) {
1113 self.deadlines.clear();
1114 for (_, outbound) in self.pending.drain() {
1115 let _ = outbound.reply.send(Err(error.clone()));
1116 }
1117 for outbound in self.queue.drain(..) {
1118 let _ = outbound.reply.send(Err(error.clone()));
1119 }
1120 self.reset_pacing();
1121 }
1122
1123 fn reset_pacing(&mut self) {
1124 self.paused_until = None;
1125 self.send_at = None;
1126 for throttle in &mut self.throttles {
1127 throttle.sent.clear();
1128 }
1129 }
1130
1131 fn fail_all(&mut self, error: &ConnectorError) {
1132 self.close_in_flight(error);
1133 self.commands.close();
1134 while let Ok(command) = self.commands.try_recv() {
1135 if let Command::Request(outbound) = command {
1136 let _ = outbound.reply.send(Err(error.clone()));
1137 }
1138 }
1139 }
1140}
1141
1142#[derive(Default)]
1143struct Throttle {
1144 sent: VecDeque<Instant>,
1145}
1146
1147impl Throttle {
1148 fn ready_at(&mut self, now: Instant, limit: u32) -> Option<Instant> {
1149 while self
1150 .sent
1151 .front()
1152 .is_some_and(|sent| now.duration_since(*sent) >= LIMIT_WINDOW)
1153 {
1154 self.sent.pop_front();
1155 }
1156 let limit = usize::try_from(limit).unwrap_or(usize::MAX);
1157 let excess = (self.sent.len() + 1).checked_sub(limit)?;
1158 let blocking = excess.checked_sub(1)?;
1159 self.sent
1160 .get(blocking)
1161 .map(|sent| after(*sent, LIMIT_WINDOW))
1162 }
1163}
1164
1165fn after(start: Instant, period: Duration) -> Instant {
1166 start.checked_add(period.min(LONGEST_WAIT)).unwrap_or(start)
1167}
1168
1169fn within(start: Instant, period: Duration) -> Option<Instant> {
1170 if period > LONGEST_WAIT {
1171 return None;
1172 }
1173 start.checked_add(period)
1174}
1175
1176async fn reached(wake: Option<Instant>) {
1177 match wake {
1178 Some(wake) => sleep_until(wake).await,
1179 None => pending().await,
1180 }
1181}
1182
1183fn over_limit(result: &Result<ProtoMessage, ConnectorError>) -> bool {
1184 matches!(
1185 result,
1186 Err(ConnectorError::Server(error)) if matches!(
1187 error.code,
1188 ErrorCode::RequestFrequencyExceeded | ErrorCode::BlockedPayloadType
1189 )
1190 )
1191}
1192
1193fn store(replays: &mut Replays, key: ReplayKey, stored: Option<Stored>) {
1194 match stored {
1195 Some(stored) => {
1196 replays.insert(key, stored);
1197 }
1198 None => {
1199 replays.remove(&key);
1200 }
1201 }
1202}
1203
1204async fn write<S: AsyncRead + AsyncWrite + Unpin>(
1205 framed: &mut Framed<S>,
1206 message: &ProtoMessage,
1207 limit: Duration,
1208) -> Result<(), TransportError> {
1209 timeout(limit, framed.send(message))
1210 .await
1211 .unwrap_or(Err(TransportError::WriteTimeout(limit)))
1212}
1213
1214async fn close<S: AsyncRead + AsyncWrite + Unpin>(framed: &mut Framed<S>, limit: Duration) {
1215 let _ = timeout(limit, framed.close()).await;
1216}
1217
1218async fn hold<F: Future>(
1219 commands: &mut mpsc::Receiver<Command>,
1220 replays: &mut Replays,
1221 reason: &TransportError,
1222 work: F,
1223) -> Result<F::Output, ConnectorError> {
1224 tokio::pin!(work);
1225 loop {
1226 tokio::select! {
1227 output = &mut work => return Ok(output),
1228 command = commands.recv() => match command {
1229 Some(Command::Request(outbound)) => {
1230 let _ = outbound.reply.send(Err(reason.clone().into()));
1231 }
1232 Some(Command::Replay(key, stored)) => store(replays, key, stored),
1233 Some(Command::Reconnect) => {}
1234 Some(Command::Shutdown) | None => return Err(ConnectorError::Closed),
1235 },
1236 }
1237 }
1238}
1239
1240async fn replay<S: AsyncRead + AsyncWrite + Unpin>(
1241 framed: &mut Framed<S>,
1242 id: String,
1243 stored: Stored,
1244 limit: Duration,
1245 events: &broadcast::Sender<Event>,
1246 pushed: &mut Vec<SessionEnded>,
1247) -> Result<Result<ProtoMessage, ConnectorError>, TransportError> {
1248 let message = ProtoMessage {
1249 payload_type: stored.payload_type,
1250 payload: Some(stored.payload),
1251 client_msg_id: Some(id.clone()),
1252 };
1253 write(framed, &message, limit).await?;
1254 Ok(
1255 match timeout(limit, answer_to(framed, &id, events, pushed)).await {
1256 Ok(answer) => outcome(answer?),
1257 Err(_) => Err(ConnectorError::Timeout(limit)),
1258 },
1259 )
1260}
1261
1262async fn answer_to<S: AsyncRead + AsyncWrite + Unpin>(
1263 framed: &mut Framed<S>,
1264 id: &str,
1265 events: &broadcast::Sender<Event>,
1266 pushed: &mut Vec<SessionEnded>,
1267) -> Result<ProtoMessage, TransportError> {
1268 loop {
1269 let message = framed.receive().await?;
1270 if message.client_msg_id.as_deref() == Some(id) {
1271 return Ok(message);
1272 }
1273 if message.payload_type != ProtoHeartbeatEvent::PAYLOAD_TYPE {
1274 pushed.extend(sessions_ended_by(&message));
1275 let _ = events.send(Event::Message(message));
1276 }
1277 }
1278}
1279
1280fn outcome(message: ProtoMessage) -> Result<ProtoMessage, ConnectorError> {
1281 if message.payload_type == ProtoOaErrorRes::PAYLOAD_TYPE {
1282 Err(ProtoOaErrorRes::from_message(&message)
1283 .map_or_else(|error| error, |response| ServerError::from(response).into()))
1284 } else if message.payload_type == ProtoErrorRes::PAYLOAD_TYPE {
1285 Err(ProtoErrorRes::from_message(&message)
1286 .map_or_else(|error| error, |response| ServerError::from(response).into()))
1287 } else {
1288 Ok(message)
1289 }
1290}
1291
1292#[cfg(test)]
1293mod tests {
1294 use std::time::Duration;
1295
1296 use super::{
1297 ConnectionOptions, DEFAULT_INBOUND_TIMEOUT, Event, LIMIT_WINDOW, LONGEST_WAIT,
1298 MAX_COMMAND_CAPACITY, MAX_EVENT_CAPACITY, ReplayKey, ReplayStage,
1299 SERVER_HEARTBEAT_INTERVAL, Throttle, after, outcome, over_limit, within,
1300 };
1301 use crate::common::config::ConfigurationOpenApi;
1302 use crate::common::errors::{ConnectorError, ErrorCode, TransportError};
1303 use crate::open_api::messages::{
1304 ProtoErrorRes, ProtoMessage, ProtoOaErrorRes, ProtoOaSpotEvent, ProtoOaVersionRes,
1305 };
1306 use crate::open_api::payload::Payload;
1307 use tokio::time::Instant;
1308
1309 #[test]
1310 fn options_take_timing_and_retries_of_the_configuration() {
1311 let configuration = ConfigurationOpenApi::builder()
1312 .client_id("id")
1313 .client_secret("secret")
1314 .access_token("token")
1315 .timeout(Duration::from_secs(3))
1316 .reconnect_delay(Duration::from_secs(2))
1317 .retries(7)
1318 .backoff(Duration::from_millis(250))
1319 .build()
1320 .expect("configuration");
1321 let options = ConnectionOptions::from(&configuration);
1322 assert_eq!(
1323 options,
1324 ConnectionOptions::new()
1325 .timeout(Duration::from_secs(3))
1326 .reconnect_delay(Duration::from_secs(2))
1327 .retries(7)
1328 .backoff(Duration::from_millis(250))
1329 );
1330 }
1331
1332 #[test]
1333 fn silence_allowed_follows_the_pace_of_the_server_unless_it_is_set() {
1334 assert_eq!(DEFAULT_INBOUND_TIMEOUT, Duration::from_secs(75));
1335 assert!(DEFAULT_INBOUND_TIMEOUT >= SERVER_HEARTBEAT_INTERVAL * 2);
1336 assert_eq!(ConnectionOptions::new().silence(), DEFAULT_INBOUND_TIMEOUT);
1337 let quick = ConnectionOptions::new()
1338 .heartbeat_interval(Duration::from_secs(2))
1339 .timeout(Duration::from_secs(1));
1340 assert_eq!(quick.silence(), DEFAULT_INBOUND_TIMEOUT);
1341 assert_eq!(
1342 quick.clone().inbound_timeout(Duration::ZERO).silence(),
1343 Duration::from_millis(1)
1344 );
1345 assert_eq!(
1346 quick.inbound_timeout(Duration::from_secs(90)).silence(),
1347 Duration::from_secs(90)
1348 );
1349 }
1350
1351 #[test]
1352 fn options_raise_degenerate_values() {
1353 let options = ConnectionOptions::new()
1354 .timeout(Duration::ZERO)
1355 .heartbeat_interval(Duration::ZERO)
1356 .command_capacity(0)
1357 .event_capacity(0)
1358 .reconnect_delay(Duration::ZERO)
1359 .reconnect_attempts(Some(4))
1360 .request_limit(0)
1361 .historical_limit(0)
1362 .retries(0)
1363 .backoff(Duration::ZERO);
1364 assert_eq!(options.timeout, Duration::from_millis(1));
1365 assert_eq!(options.heartbeat_interval, Duration::from_millis(1));
1366 assert_eq!(options.command_capacity, 1);
1367 assert_eq!(options.event_capacity, 1);
1368 let largest = ConnectionOptions::new()
1369 .command_capacity(usize::MAX)
1370 .event_capacity(usize::MAX);
1371 assert_eq!(largest.command_capacity, MAX_COMMAND_CAPACITY);
1372 assert_eq!(largest.event_capacity, MAX_EVENT_CAPACITY);
1373 assert_eq!(options.reconnect_delay, Duration::from_millis(1));
1374 assert_eq!(options.reconnect_attempts, Some(4));
1375 assert_eq!(options.request_limit, 1);
1376 assert_eq!(options.historical_limit, 1);
1377 assert_eq!(options.retries, 0);
1378 assert_eq!(options.backoff, Duration::from_millis(1));
1379 assert_eq!(ConnectionOptions::new().reconnect_attempts, None);
1380 }
1381
1382 #[test]
1383 fn error_responses_become_server_errors() {
1384 let open_api = ProtoOaErrorRes {
1385 error_code: "CH_CLIENT_AUTH_FAILURE".to_owned(),
1386 ..ProtoOaErrorRes::default()
1387 };
1388 assert!(matches!(
1389 outcome(open_api.to_message(None)),
1390 Err(ConnectorError::Server(error)) if error.code == ErrorCode::ChClientAuthFailure
1391 ));
1392 let common = ProtoErrorRes {
1393 error_code: "UNSUPPORTED_MESSAGE".to_owned(),
1394 ..ProtoErrorRes::default()
1395 };
1396 assert!(matches!(
1397 outcome(common.to_message(None)),
1398 Err(ConnectorError::Server(error)) if error.code == ErrorCode::UnsupportedMessage
1399 ));
1400 for payload_type in [50, 2142] {
1401 let malformed = ProtoMessage {
1402 payload_type,
1403 payload: Some(vec![0xff]),
1404 client_msg_id: None,
1405 };
1406 assert!(matches!(
1407 outcome(malformed),
1408 Err(ConnectorError::Transport(TransportError::Decode(_)))
1409 ));
1410 }
1411 let fine = ProtoOaVersionRes::default().to_message(None);
1412 assert_eq!(outcome(fine.clone()), Ok(fine));
1413 }
1414
1415 #[test]
1416 fn event_decodes_only_the_message_it_carries() {
1417 let spot = ProtoOaSpotEvent {
1418 ctid_trader_account_id: 1,
1419 symbol_id: 2,
1420 ..ProtoOaSpotEvent::default()
1421 };
1422 let event = Event::Message(spot.to_message(None));
1423 assert_eq!(event.decode::<ProtoOaSpotEvent>(), Some(spot));
1424 assert_eq!(event.decode::<ProtoOaVersionRes>(), None);
1425 let lost = Event::Disconnected(TransportError::Closed);
1426 assert_eq!(lost.decode::<ProtoOaSpotEvent>(), None);
1427 let again = Event::Reconnected { attempt: 1 };
1428 assert_eq!(again.decode::<ProtoOaSpotEvent>(), None);
1429 }
1430
1431 #[test]
1432 fn replay_keys_order_by_stage_then_name() {
1433 let mut keys = [
1434 ReplayKey::new(ReplayStage::Subscription, "a"),
1435 ReplayKey::new(ReplayStage::Account, "2"),
1436 ReplayKey::new(ReplayStage::Application, "z"),
1437 ReplayKey::new(ReplayStage::Account, "1"),
1438 ];
1439 keys.sort();
1440 let order: Vec<(ReplayStage, &str)> =
1441 keys.iter().map(|key| (key.stage(), key.id())).collect();
1442 assert_eq!(
1443 order,
1444 vec![
1445 (ReplayStage::Application, "z"),
1446 (ReplayStage::Account, "1"),
1447 (ReplayStage::Account, "2"),
1448 (ReplayStage::Subscription, "a"),
1449 ]
1450 );
1451 }
1452
1453 #[test]
1454 fn throttle_names_the_moment_the_next_request_may_go() {
1455 let start = Instant::now();
1456 let mut throttle = Throttle::default();
1457 assert_eq!(throttle.ready_at(start, 2), None);
1458 throttle.sent.push_back(start);
1459 assert_eq!(throttle.ready_at(start, 2), None);
1460 throttle.sent.push_back(start + Duration::from_millis(10));
1461 assert_eq!(throttle.ready_at(start, 2), Some(start + LIMIT_WINDOW));
1462 throttle.sent.push_back(start + Duration::from_millis(20));
1463 assert_eq!(
1464 throttle.ready_at(start, 2),
1465 Some(start + Duration::from_millis(10) + LIMIT_WINDOW)
1466 );
1467 let later = start + LIMIT_WINDOW + Duration::from_millis(10);
1468 assert_eq!(throttle.ready_at(later, 2), None);
1469 assert_eq!(throttle.sent.len(), 1);
1470 }
1471
1472 #[test]
1473 fn waits_too_long_for_the_clock_end_at_the_longest_wait() {
1474 let start = Instant::now();
1475 let brief = Duration::from_secs(90);
1476 assert_eq!(after(start, brief), start + brief);
1477 assert_eq!(after(start, LONGEST_WAIT), start + LONGEST_WAIT);
1478 assert_eq!(after(start, Duration::MAX), start + LONGEST_WAIT);
1479 let far = after(start, Duration::MAX);
1480 assert_eq!(after(far, Duration::MAX), far + LONGEST_WAIT);
1481 assert_eq!(within(start, brief), Some(start + brief));
1482 assert_eq!(within(start, LONGEST_WAIT), Some(start + LONGEST_WAIT));
1483 assert_eq!(within(start, LONGEST_WAIT + Duration::from_nanos(1)), None);
1484 assert_eq!(within(start, Duration::MAX), None);
1485 let options = ConnectionOptions::new()
1486 .timeout(Duration::MAX)
1487 .heartbeat_interval(Duration::MAX);
1488 assert_eq!(options.silence(), DEFAULT_INBOUND_TIMEOUT);
1489 assert_eq!(
1490 options.inbound_timeout(Duration::MAX).silence(),
1491 Duration::MAX
1492 );
1493 let mut throttle = Throttle::default();
1494 throttle.sent.push_back(far);
1495 assert_eq!(throttle.ready_at(far, 1), Some(far + LIMIT_WINDOW));
1496 }
1497
1498 #[test]
1499 fn only_limit_errors_are_retried() {
1500 let refusal = |code: &str| {
1501 outcome(
1502 ProtoOaErrorRes {
1503 error_code: code.to_owned(),
1504 ..ProtoOaErrorRes::default()
1505 }
1506 .to_message(None),
1507 )
1508 };
1509 assert!(over_limit(&refusal("REQUEST_FREQUENCY_EXCEEDED")));
1510 assert!(over_limit(&refusal("BLOCKED_PAYLOAD_TYPE")));
1511 assert!(!over_limit(&refusal("INVALID_REQUEST")));
1512 assert!(!over_limit(&Ok(
1513 ProtoOaVersionRes::default().to_message(None)
1514 )));
1515 assert!(!over_limit(&Err(ConnectorError::Closed)));
1516 }
1517}