Skip to main content

ctrader_connector/open_api/
connection.rs

1//! Connection task and the handle callers talk to it through.
2//!
3//! One task owns the socket. Callers hold a cloneable [`Connection`]; a request travels over a
4//! bounded command channel, is framed with a fresh `clientMsgId` and resolves when the response
5//! carrying the same id arrives. Every id comes from one counter that requests and replayed
6//! requests share and that only ever grows, so no id is used twice on a connection and a late
7//! answer to a request given up is never taken for the answer to another one. Messages the
8//! server pushes on its own are fanned out as [`Event`]s. The task sends a heartbeat whenever
9//! nothing was written for the heartbeat interval, which keeps the server from closing an idle
10//! connection.
11//!
12//! Deadlines: nothing the task waits for on the socket is open-ended. A write that does not
13//! complete within the time allowed for a response fails with
14//! [`TransportError::WriteTimeout`], and a server that sends nothing at all, not even a
15//! heartbeat, for the period set with [`ConnectionOptions::inbound_timeout`] fails the
16//! connection with [`TransportError::Idle`]. Both count as a loss of the connection.
17//! Every duration an option carries is added to the clock without overflow. A heartbeat
18//! interval, a silence allowed or a time allowed for a response of more than thirty years is
19//! never due, so [`Duration::MAX`] stands for never and turns the wait it is given to off. A
20//! pause before a request is sent again ends thirty years ahead at the latest.
21//!
22//! Backpressure: the command channel is bounded, so callers wait while the task is busy, and the
23//! event stream is a bounded broadcast, so a subscriber that falls behind skips the oldest
24//! events instead of stalling the task.
25//!
26//! Limits: the server accepts a number of requests per second on one connection, fewer for
27//! historical data. The task holds requests back so that neither limit is exceeded, and when
28//! the server still answers that a limit was exceeded it pauses and sends the request again, up
29//! to the configured number of retries.
30//!
31//! Reconnection: when the socket is lost the task fails the requests in flight, reports
32//! [`Event::Disconnected`] and connects again, waiting twice as long before every further
33//! attempt. Once connected it sends the registered requests again in the order of their
34//! [`ReplayStage`] (application, accounts, subscriptions) and then reports
35//! [`Event::Reconnected`]. The registered requests are read one at a time, each just before it
36//! is sent, so a registration removed meanwhile is not replayed and one added meanwhile is.
37//! Requests made while the connection is down fail at once with the reason of the loss.
38//!
39//! End of an authorisation: the server can end the authorisation of an account, or of the
40//! application, while the connection stays up: it announces that an access token is no longer
41//! valid, that it dropped the session of an account, or that it ends the session of the
42//! application. The same holds when it refuses the access token while an account is authorised
43//! again after a reconnection. The task then reports [`Event::SessionEnded`], forgets the
44//! registered requests of the session, so that nothing is replayed with a dead token, and
45//! answers every later request of the session with [`ConnectorError::SessionEnded`] without
46//! sending it. The session is usable again once it is authorised anew on this connection: an
47//! account with a valid access token, the application with its credentials followed by its
48//! accounts.
49//!
50//! Requests that must not arrive twice: a request made with [`Connection::request_once`], an
51//! order for one, is written at most once. It is neither sent again after a limit error nor
52//! replayed, and when the connection fails or the time allowed runs out after it was written
53//! the caller is told that its outcome is unknown, with [`ConnectorError::OutcomeUnknown`].
54
55use std::collections::{BTreeMap, HashMap, VecDeque};
56use std::future::pending;
57use std::sync::atomic::{AtomicU64, Ordering};
58use std::sync::{Arc, Mutex, PoisonError};
59use std::time::Duration;
60
61use tokio::io::{AsyncRead, AsyncWrite};
62use tokio::sync::{broadcast, mpsc, oneshot};
63use tokio::task::JoinHandle;
64use tokio::time::{Instant, sleep, sleep_until, timeout};
65
66use crate::common::config::{
67    ConfigurationOpenApi, DEFAULT_BACKOFF, DEFAULT_RECONNECT_DELAY, DEFAULT_RETRIES,
68    DEFAULT_TIMEOUT,
69};
70use crate::common::errors::{
71    ConnectorError, ErrorCode, Interruption, ServerError, SessionEndCause, SessionEnded,
72    TransportError,
73};
74use crate::common::ids::AccountId;
75use crate::open_api::messages::{
76    ProtoErrorRes, ProtoHeartbeatEvent, ProtoMessage, ProtoOaAccountAuthReq, ProtoOaAccountAuthRes,
77    ProtoOaApplicationAuthReq, ProtoOaApplicationAuthRes, ProtoOaErrorRes,
78};
79use crate::open_api::payload::{Payload, Request, account_of, is_historical, sessions_ended_by};
80use crate::open_api::transport::{Connect, Framed};
81
82/// Pause after the last write before a heartbeat is sent.
83pub const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(10);
84
85/// Number of requests that may wait in the command channel.
86pub const DEFAULT_COMMAND_CAPACITY: usize = 64;
87
88/// Number of events kept for a subscriber that falls behind.
89pub const DEFAULT_EVENT_CAPACITY: usize = 1024;
90
91/// Largest number of requests that may wait in the command channel; a larger number asked for
92/// is lowered to it.
93pub const MAX_COMMAND_CAPACITY: usize = 65_536;
94
95/// Largest number of events kept for a subscriber that falls behind; a larger number asked for
96/// is lowered to it, because the room for the events is set aside when the connection opens.
97pub const MAX_EVENT_CAPACITY: usize = 65_536;
98
99/// Requests the server accepts per second on one connection, historical data excluded.
100pub const REQUEST_LIMIT: u32 = 50;
101
102/// Requests for historical data the server accepts per second on one connection.
103pub const HISTORICAL_LIMIT: u32 = 5;
104
105/// Longest pause after the server answered that a limit was exceeded.
106pub const BACKOFF_LIMIT: Duration = Duration::from_secs(30);
107
108const LIMIT_WINDOW: Duration = Duration::from_millis(1100);
109
110/// Longest pause between two reconnection attempts.
111pub const RECONNECT_DELAY_LIMIT: Duration = Duration::from_secs(60);
112
113const SHORTEST_INTERVAL: Duration = Duration::from_millis(1);
114
115/// Pause between two heartbeats the server sends on a connection that is otherwise quiet.
116pub const SERVER_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(30);
117
118/// Silence of the server after which the connection counts as lost unless another period is
119/// set: two and a half times [`SERVER_HEARTBEAT_INTERVAL`], so one heartbeat of the server may
120/// go missing, and the next one be late by half an interval, before a connection is given up.
121pub const DEFAULT_INBOUND_TIMEOUT: Duration = Duration::from_secs(75);
122
123const LONGEST_WAIT: Duration = Duration::from_secs(30 * 365 * 86_400);
124
125macro_rules! log_debug {
126    ($($argument:tt)*) => {{
127        #[cfg(feature = "logging")]
128        tracing::debug!($($argument)*);
129    }};
130}
131
132/// Tunable behaviour of a [`Connection`].
133#[derive(Debug, Clone, PartialEq, Eq)]
134#[must_use]
135pub struct ConnectionOptions {
136    timeout: Duration,
137    heartbeat_interval: Duration,
138    command_capacity: usize,
139    event_capacity: usize,
140    reconnect_delay: Duration,
141    reconnect_attempts: Option<u32>,
142    request_limit: u32,
143    historical_limit: u32,
144    retries: u32,
145    backoff: Duration,
146    inbound_timeout: Option<Duration>,
147}
148
149impl Default for ConnectionOptions {
150    fn default() -> Self {
151        Self {
152            timeout: DEFAULT_TIMEOUT,
153            heartbeat_interval: HEARTBEAT_INTERVAL,
154            command_capacity: DEFAULT_COMMAND_CAPACITY,
155            event_capacity: DEFAULT_EVENT_CAPACITY,
156            reconnect_delay: DEFAULT_RECONNECT_DELAY,
157            reconnect_attempts: None,
158            request_limit: REQUEST_LIMIT,
159            historical_limit: HISTORICAL_LIMIT,
160            retries: DEFAULT_RETRIES,
161            backoff: DEFAULT_BACKOFF,
162            inbound_timeout: None,
163        }
164    }
165}
166
167impl From<&ConfigurationOpenApi> for ConnectionOptions {
168    fn from(configuration: &ConfigurationOpenApi) -> Self {
169        Self::default()
170            .timeout(configuration.timeout())
171            .reconnect_delay(configuration.reconnect_delay())
172            .retries(configuration.retries())
173            .backoff(configuration.backoff())
174    }
175}
176
177impl ConnectionOptions {
178    /// Starts from the default options.
179    pub fn new() -> Self {
180        Self::default()
181    }
182
183    /// Sets the time allowed between sending a request and receiving its response.
184    ///
185    /// A duration below one millisecond is raised to one millisecond.
186    pub fn timeout(mut self, timeout: Duration) -> Self {
187        self.timeout = timeout.max(SHORTEST_INTERVAL);
188        self
189    }
190
191    /// Sets the pause after the last write before a heartbeat is sent.
192    ///
193    /// A duration below one millisecond is raised to one millisecond.
194    pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
195        self.heartbeat_interval = interval.max(SHORTEST_INTERVAL);
196        self
197    }
198
199    /// Sets the number of requests that may wait in the command channel.
200    ///
201    /// Zero is raised to one and a number above [`MAX_COMMAND_CAPACITY`] is lowered to it.
202    pub fn command_capacity(mut self, capacity: usize) -> Self {
203        self.command_capacity = capacity.clamp(1, MAX_COMMAND_CAPACITY);
204        self
205    }
206
207    /// Sets the number of events kept for a subscriber that falls behind.
208    ///
209    /// Zero is raised to one and a number above [`MAX_EVENT_CAPACITY`] is lowered to it.
210    pub fn event_capacity(mut self, capacity: usize) -> Self {
211        self.event_capacity = capacity.clamp(1, MAX_EVENT_CAPACITY);
212        self
213    }
214
215    /// Sets the pause before the first reconnection attempt; it doubles before every further
216    /// attempt up to [`RECONNECT_DELAY_LIMIT`].
217    ///
218    /// A duration below one millisecond is raised to one millisecond.
219    pub fn reconnect_delay(mut self, delay: Duration) -> Self {
220        self.reconnect_delay = delay.max(SHORTEST_INTERVAL);
221        self
222    }
223
224    /// Sets how many times in a row the task tries to connect again before it gives up.
225    ///
226    /// `None` keeps trying, which is the default; `Some(0)` turns reconnection off.
227    pub const fn reconnect_attempts(mut self, attempts: Option<u32>) -> Self {
228        self.reconnect_attempts = attempts;
229        self
230    }
231
232    /// Sets the number of requests sent per second, historical data excluded; at least one.
233    pub fn request_limit(mut self, limit: u32) -> Self {
234        self.request_limit = limit.max(1);
235        self
236    }
237
238    /// Sets the number of requests for historical data sent per second; at least one.
239    pub fn historical_limit(mut self, limit: u32) -> Self {
240        self.historical_limit = limit.max(1);
241        self
242    }
243
244    /// Sets how often a request is sent again after the server answered that a limit was
245    /// exceeded; zero hands the error to the caller at once.
246    pub const fn retries(mut self, retries: u32) -> Self {
247        self.retries = retries;
248        self
249    }
250
251    /// Sets the pause after the server answered that a limit was exceeded; it doubles with
252    /// every retry of the same request up to [`BACKOFF_LIMIT`].
253    ///
254    /// A duration below one millisecond is raised to one millisecond.
255    pub fn backoff(mut self, backoff: Duration) -> Self {
256        self.backoff = backoff.max(SHORTEST_INTERVAL);
257        self
258    }
259
260    /// Sets how long the server may stay silent before the connection counts as lost.
261    ///
262    /// Every frame the server sends, heartbeats included, starts the period anew. When it runs
263    /// out the task reports [`Event::Disconnected`] with [`TransportError::Idle`] and connects
264    /// again. Left unset, the period is [`DEFAULT_INBOUND_TIMEOUT`]. It follows from the pace of
265    /// the server alone, which sends a heartbeat of its own every
266    /// [`SERVER_HEARTBEAT_INTERVAL`] on a connection that is otherwise quiet, and from nothing
267    /// the client is set to do: neither the heartbeat interval nor the time allowed for a
268    /// response changes it. A period shorter than the pace of the server gives up connections
269    /// that are in order.
270    ///
271    /// A duration below one millisecond is raised to one millisecond. [`Duration::MAX`] turns
272    /// the watch off: the server may then stay silent for as long as it likes.
273    pub fn inbound_timeout(mut self, period: Duration) -> Self {
274        self.inbound_timeout = Some(period.max(SHORTEST_INTERVAL));
275        self
276    }
277
278    fn silence(&self) -> Duration {
279        self.inbound_timeout.unwrap_or(DEFAULT_INBOUND_TIMEOUT)
280    }
281}
282
283/// Something that happened on the connection without a request asking for it.
284#[derive(Debug, Clone, PartialEq, Eq)]
285#[non_exhaustive]
286pub enum Event {
287    /// The server sent a message that answers no pending request.
288    Message(ProtoMessage),
289    /// The connection was lost for the given reason; reconnection follows.
290    Disconnected(TransportError),
291    /// An attempt to connect again failed; another one follows unless the limit is reached.
292    ReconnectFailed {
293        /// Number of the attempt since the loss, starting at one.
294        attempt: u32,
295        /// Reason the attempt failed.
296        reason: TransportError,
297    },
298    /// The connection is established again and the registered requests were replayed.
299    Reconnected {
300        /// Number of the attempt that succeeded, starting at one.
301        attempt: u32,
302    },
303    /// A registered request was refused or left unanswered while it was replayed.
304    ReplayFailed {
305        /// Key the request was registered under.
306        key: ReplayKey,
307        /// Failure of the replayed request.
308        error: ConnectorError,
309    },
310    /// The server ended the authorisation of a session; its requests fail with
311    /// [`ConnectorError::SessionEnded`] until it is authorised again.
312    SessionEnded(SessionEnded),
313}
314
315impl Event {
316    /// Returns the pushed message when the event carries one of type `M`.
317    #[must_use]
318    pub fn decode<M: Payload>(&self) -> Option<M> {
319        match self {
320            Self::Message(message) => M::from_message(message).ok(),
321            _ => None,
322        }
323    }
324}
325
326/// Counters of a connection since it was opened.
327#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
328#[non_exhaustive]
329pub struct Stats {
330    /// Requests written to the socket.
331    pub requests_sent: u64,
332    /// Heartbeats written to the socket.
333    pub heartbeats_sent: u64,
334    /// Heartbeats received from the server.
335    pub heartbeats_received: u64,
336    /// Times the connection was established again after a loss.
337    pub reconnects: u64,
338    /// Requests sent again after the server answered that a limit was exceeded.
339    pub throttle_retries: u64,
340}
341
342#[derive(Debug, Default)]
343struct Counters {
344    requests_sent: AtomicU64,
345    heartbeats_sent: AtomicU64,
346    heartbeats_received: AtomicU64,
347    reconnects: AtomicU64,
348    throttle_retries: AtomicU64,
349}
350
351fn bump(counter: &AtomicU64) {
352    counter.fetch_add(1, Ordering::Relaxed);
353}
354
355/// Phase of the restoration a replayed request belongs to; earlier phases are replayed first.
356#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
357#[non_exhaustive]
358pub enum ReplayStage {
359    /// Authorisation of the application.
360    Application,
361    /// Authorisation of a trading account.
362    Account,
363    /// Subscription to a stream of events.
364    Subscription,
365}
366
367/// Name a request is registered under for replay after a reconnection.
368///
369/// Registering another request under the same key replaces the earlier one.
370#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
371pub struct ReplayKey {
372    stage: ReplayStage,
373    id: String,
374}
375
376impl ReplayKey {
377    /// Builds a key from the phase and a name unique within it.
378    #[must_use]
379    pub fn new(stage: ReplayStage, id: impl Into<String>) -> Self {
380        Self {
381            stage,
382            id: id.into(),
383        }
384    }
385
386    /// Returns the phase the request is replayed in.
387    #[must_use]
388    pub const fn stage(&self) -> ReplayStage {
389        self.stage
390    }
391
392    /// Returns the name of the request within its phase.
393    #[must_use]
394    pub fn id(&self) -> &str {
395        &self.id
396    }
397}
398
399#[derive(Clone, PartialEq, Eq)]
400struct Stored {
401    payload_type: u32,
402    payload: Vec<u8>,
403}
404
405type Replays = BTreeMap<ReplayKey, Stored>;
406
407type Reply = oneshot::Sender<Result<ProtoMessage, ConnectorError>>;
408
409struct Outbound {
410    payload_type: u32,
411    payload: Vec<u8>,
412    historical: bool,
413    once: bool,
414    attempts: u32,
415    reply: Reply,
416}
417
418impl Outbound {
419    fn interrupted(&self, by: Interruption) -> ConnectorError {
420        if self.once {
421            ConnectorError::OutcomeUnknown(by)
422        } else {
423            match by {
424                Interruption::Timeout(limit) => ConnectorError::Timeout(limit),
425                Interruption::Transport(reason) => ConnectorError::Transport(reason),
426                Interruption::Skipped => ConnectorError::OutcomeUnknown(by),
427            }
428        }
429    }
430}
431
432#[derive(Default)]
433struct Ended {
434    application: Option<SessionEndCause>,
435    accounts: BTreeMap<AccountId, SessionEndCause>,
436}
437
438impl Ended {
439    fn refusal(&self, payload_type: u32, payload: &[u8]) -> Option<SessionEnded> {
440        if payload_type == ProtoOaApplicationAuthReq::PAYLOAD_TYPE {
441            return None;
442        }
443        if let Some(cause) = &self.application {
444            return Some(SessionEnded::new(None, cause.clone()));
445        }
446        if payload_type == ProtoOaAccountAuthReq::PAYLOAD_TYPE {
447            return None;
448        }
449        let account = account_of(payload_type, payload)?;
450        let cause = self.accounts.get(&account)?;
451        Some(SessionEnded::new(Some(account), cause.clone()))
452    }
453
454    fn granted(&mut self, answer: &ProtoMessage) {
455        if answer.payload_type == ProtoOaApplicationAuthRes::PAYLOAD_TYPE {
456            self.application = None;
457        } else if answer.payload_type == ProtoOaAccountAuthRes::PAYLOAD_TYPE {
458            let payload = answer.payload.as_deref().unwrap_or_default();
459            if let Some(account) = account_of(answer.payload_type, payload) {
460                self.accounts.remove(&account);
461            }
462        }
463    }
464}
465
466fn token_refused(key: &ReplayKey, stored: &Stored, error: &ConnectorError) -> Option<SessionEnded> {
467    let ConnectorError::Server(error) = error else {
468        return None;
469    };
470    let dead = matches!(
471        error.code,
472        ErrorCode::ChAccessTokenInvalid | ErrorCode::OaAuthTokenExpired
473    );
474    if !dead || key.stage() != ReplayStage::Account {
475        return None;
476    }
477    let account = account_of(stored.payload_type, &stored.payload)?;
478    Some(SessionEnded::new(
479        Some(account),
480        SessionEndCause::TokenRefused(error.code.clone()),
481    ))
482}
483
484enum Command {
485    Request(Outbound),
486    Replay(ReplayKey, Option<Stored>),
487    Reconnect,
488    Shutdown,
489}
490
491/// Cloneable handle of a connection task.
492///
493/// The task ends when [`Connection::shutdown`] is called or the last handle is dropped.
494#[derive(Debug, Clone)]
495pub struct Connection {
496    commands: mpsc::Sender<Command>,
497    events: broadcast::Sender<Event>,
498    counters: Arc<Counters>,
499    task: Arc<Mutex<Option<JoinHandle<()>>>>,
500}
501
502impl Connection {
503    /// Connects to `host` on `port` and starts the connection task.
504    ///
505    /// Must be called from within a Tokio runtime.
506    ///
507    /// # Errors
508    ///
509    /// Returns [`ConnectorError::Transport`] when the connection cannot be established.
510    pub async fn open<C: Connect>(
511        connector: C,
512        host: &str,
513        port: u16,
514        options: ConnectionOptions,
515    ) -> Result<Self, ConnectorError> {
516        let stream = connector.connect(host, port).await?;
517        log_debug!(host, port, "connected");
518        let (commands, receiver) = mpsc::channel(options.command_capacity);
519        let (events, _) = broadcast::channel(options.event_capacity);
520        let counters = Arc::new(Counters::default());
521        let actor = Actor {
522            connector,
523            host: host.to_owned(),
524            port,
525            heartbeat_at: within(Instant::now(), options.heartbeat_interval),
526            silent_at: within(Instant::now(), options.silence()),
527            options,
528            commands: receiver,
529            events: events.clone(),
530            counters: Arc::clone(&counters),
531            queue: VecDeque::new(),
532            pending: HashMap::new(),
533            deadlines: VecDeque::new(),
534            replays: Replays::new(),
535            ended: Ended::default(),
536            throttles: [Throttle::default(), Throttle::default()],
537            paused_until: None,
538            send_at: None,
539            next_id: 0,
540        };
541        let task = tokio::spawn(actor.run(Framed::new(stream)));
542        Ok(Self {
543            commands,
544            events,
545            counters,
546            task: Arc::new(Mutex::new(Some(task))),
547        })
548    }
549
550    /// Sends a request and waits for its response.
551    ///
552    /// # Errors
553    ///
554    /// Returns [`ConnectorError::Server`] when the server answers with an error,
555    /// [`ConnectorError::Timeout`] when it does not answer in time,
556    /// [`ConnectorError::Transport`] when the connection fails while the request is pending,
557    /// [`ConnectorError::UnexpectedResponse`] when the answer has another type, and
558    /// [`ConnectorError::Closed`] when the connection is shut down.
559    pub async fn request<R: Request>(&self, request: &R) -> Result<R::Response, ConnectorError> {
560        let message = self
561            .send(
562                R::PAYLOAD_TYPE,
563                request.encode_to_vec(),
564                R::HISTORICAL,
565                false,
566            )
567            .await?;
568        R::Response::from_message(&message)
569    }
570
571    /// Sends an already encoded request and waits for the envelope of its response.
572    ///
573    /// # Errors
574    ///
575    /// Fails like [`Connection::request`], except that the type of the answer is not checked.
576    pub async fn request_message(
577        &self,
578        payload_type: u32,
579        payload: Vec<u8>,
580    ) -> Result<ProtoMessage, ConnectorError> {
581        self.send(payload_type, payload, is_historical(payload_type), false)
582            .await
583    }
584
585    /// Sends an already encoded request that the server must not receive twice, such as an
586    /// order, and waits for the envelope of the first message that answers it.
587    ///
588    /// The request is written to the socket at most once. It is not sent again when the server
589    /// answers that a limit was exceeded, which is handed to the caller as the refusal it is,
590    /// and it is never replayed after a reconnection. The answer is also reported to the
591    /// subscribers of [`Connection::events`] as an [`Event::Message`], ahead of the messages
592    /// the server sends after it, so that a reader of the events sees all the server said about
593    /// the request in the order it was said.
594    ///
595    /// # Errors
596    ///
597    /// Returns [`ConnectorError::OutcomeUnknown`] when the request was written and the
598    /// connection failed or the time allowed ran out before an answer arrived, in which case
599    /// the server may or may not have acted on it. [`ConnectorError::Transport`] means the
600    /// request was not written: the connection was down when it was made or was lost while it
601    /// waited its turn. [`ConnectorError::Closed`] is returned when the connection is shut down
602    /// from this side, also while the answer is awaited. Fails like
603    /// [`Connection::request_message`] otherwise.
604    pub async fn request_once(
605        &self,
606        payload_type: u32,
607        payload: Vec<u8>,
608    ) -> Result<ProtoMessage, ConnectorError> {
609        self.send(payload_type, payload, is_historical(payload_type), true)
610            .await
611    }
612
613    async fn send(
614        &self,
615        payload_type: u32,
616        payload: Vec<u8>,
617        historical: bool,
618        once: bool,
619    ) -> Result<ProtoMessage, ConnectorError> {
620        let (reply, response) = oneshot::channel();
621        let outbound = Outbound {
622            payload_type,
623            payload,
624            reply,
625            historical,
626            once,
627            attempts: 0,
628        };
629        self.commands
630            .send(Command::Request(outbound))
631            .await
632            .map_err(|_| ConnectorError::Closed)?;
633        response.await.map_err(|_| ConnectorError::Closed)?
634    }
635
636    /// Registers a request to be sent again, under `key`, after every reconnection.
637    ///
638    /// The request is not sent now. Requests are replayed in the order of their keys. A request
639    /// registered or replaced while the connection is being restored is replayed before
640    /// [`Event::Reconnected`] is reported.
641    ///
642    /// # Errors
643    ///
644    /// Returns [`ConnectorError::Closed`] when the connection is shut down.
645    pub async fn register_replay<R: Request>(
646        &self,
647        key: ReplayKey,
648        request: &R,
649    ) -> Result<(), ConnectorError> {
650        let stored = Stored {
651            payload_type: R::PAYLOAD_TYPE,
652            payload: request.encode_to_vec(),
653        };
654        self.command(Command::Replay(key, Some(stored))).await
655    }
656
657    /// Forgets the request registered under `key`.
658    ///
659    /// While the connection is being restored the registered requests are looked up one at a
660    /// time, each just before it is sent, so a request removed before its turn is not replayed.
661    /// A request that was replayed already stays in force on the server: the task knows no
662    /// request that would withdraw it, which is left to the caller once
663    /// [`Event::Reconnected`] is reported.
664    ///
665    /// # Errors
666    ///
667    /// Returns [`ConnectorError::Closed`] when the connection is shut down.
668    pub async fn remove_replay(&self, key: ReplayKey) -> Result<(), ConnectorError> {
669        self.command(Command::Replay(key, None)).await
670    }
671
672    /// Drops the socket on this side, which starts a reconnection as a loss would.
673    ///
674    /// # Errors
675    ///
676    /// Returns [`ConnectorError::Closed`] when the connection is shut down.
677    pub async fn force_reconnect(&self) -> Result<(), ConnectorError> {
678        self.command(Command::Reconnect).await
679    }
680
681    async fn command(&self, command: Command) -> Result<(), ConnectorError> {
682        self.commands
683            .send(command)
684            .await
685            .map_err(|_| ConnectorError::Closed)
686    }
687
688    /// Subscribes to the events of the connection, starting with the next one.
689    ///
690    /// A subscriber that falls more than the event capacity behind skips the oldest events and
691    /// is told how many were skipped.
692    #[must_use]
693    pub fn events(&self) -> broadcast::Receiver<Event> {
694        self.events.subscribe()
695    }
696
697    /// Waits until the connection task has ended, after which every request fails with
698    /// [`ConnectorError::Closed`].
699    pub async fn closed(&self) {
700        self.commands.closed().await;
701    }
702
703    /// Returns the counters of the connection.
704    #[must_use]
705    pub fn stats(&self) -> Stats {
706        Stats {
707            requests_sent: self.counters.requests_sent.load(Ordering::Relaxed),
708            heartbeats_sent: self.counters.heartbeats_sent.load(Ordering::Relaxed),
709            heartbeats_received: self.counters.heartbeats_received.load(Ordering::Relaxed),
710            reconnects: self.counters.reconnects.load(Ordering::Relaxed),
711            throttle_retries: self.counters.throttle_retries.load(Ordering::Relaxed),
712        }
713    }
714
715    /// Stops the connection task and waits until the socket is closed.
716    ///
717    /// Requests still pending fail with [`ConnectorError::Closed`]. Calling it again is
718    /// harmless.
719    pub async fn shutdown(&self) {
720        let _ = self.commands.send(Command::Shutdown).await;
721        let task = self
722            .task
723            .lock()
724            .unwrap_or_else(PoisonError::into_inner)
725            .take();
726        if let Some(task) = task {
727            let _ = task.await;
728        }
729    }
730}
731
732#[derive(Debug)]
733enum Exit {
734    Shutdown,
735    Lost(TransportError),
736}
737
738struct Actor<C: Connect> {
739    connector: C,
740    host: String,
741    port: u16,
742    options: ConnectionOptions,
743    commands: mpsc::Receiver<Command>,
744    events: broadcast::Sender<Event>,
745    counters: Arc<Counters>,
746    queue: VecDeque<Outbound>,
747    pending: HashMap<String, Outbound>,
748    deadlines: VecDeque<(Option<Instant>, String)>,
749    replays: Replays,
750    ended: Ended,
751    throttles: [Throttle; 2],
752    paused_until: Option<Instant>,
753    send_at: Option<Instant>,
754    next_id: u64,
755    heartbeat_at: Option<Instant>,
756    silent_at: Option<Instant>,
757}
758
759impl<C: Connect> Actor<C> {
760    async fn run(mut self, mut framed: Framed<C::Stream>) {
761        let error = loop {
762            match self.serve(&mut framed).await {
763                Exit::Shutdown => break ConnectorError::Closed,
764                Exit::Lost(reason) => {
765                    log_debug!(%reason, "connection lost");
766                    close(&mut framed, self.options.timeout).await;
767                    self.fail_in_flight(&reason);
768                    let _ = self.events.send(Event::Disconnected(reason.clone()));
769                    match self.reconnect(reason).await {
770                        Ok(fresh) => framed = fresh,
771                        Err(error) => break error,
772                    }
773                }
774            }
775        };
776        self.fail_all(&error);
777        close(&mut framed, self.options.timeout).await;
778    }
779
780    async fn reconnect(
781        &mut self,
782        reason: TransportError,
783    ) -> Result<Framed<C::Stream>, ConnectorError> {
784        let mut last = reason;
785        let mut delay = self.options.reconnect_delay;
786        let longest = RECONNECT_DELAY_LIMIT.max(delay);
787        let mut attempt = 0_u32;
788        loop {
789            if self
790                .options
791                .reconnect_attempts
792                .is_some_and(|limit| attempt >= limit)
793            {
794                return Err(ConnectorError::Transport(last));
795            }
796            attempt = attempt.saturating_add(1);
797            hold(&mut self.commands, &mut self.replays, &last, sleep(delay)).await?;
798            delay = delay.saturating_mul(2).min(longest);
799            match self.restore(&last).await? {
800                Ok((framed, replays_sent)) => {
801                    log_debug!(attempt, "reconnected");
802                    bump(&self.counters.reconnects);
803                    let seeded = replays_sent
804                        .min(usize::try_from(self.options.request_limit).unwrap_or(usize::MAX));
805                    if let Some(throttle) = self.throttles.first_mut() {
806                        throttle
807                            .sent
808                            .extend(std::iter::repeat_n(Instant::now(), seeded));
809                    }
810                    self.heartbeat_at = within(Instant::now(), self.options.heartbeat_interval);
811                    self.silent_at = within(Instant::now(), self.options.silence());
812                    let _ = self.events.send(Event::Reconnected { attempt });
813                    return Ok(framed);
814                }
815                Err(reason) => {
816                    log_debug!(attempt, %reason, "reconnection failed");
817                    let _ = self.events.send(Event::ReconnectFailed {
818                        attempt,
819                        reason: reason.clone(),
820                    });
821                    last = reason;
822                }
823            }
824        }
825    }
826
827    async fn restore(
828        &mut self,
829        reason: &TransportError,
830    ) -> Result<Result<(Framed<C::Stream>, usize), TransportError>, ConnectorError> {
831        let limit = self.options.timeout;
832        let spacing = LIMIT_WINDOW / self.options.request_limit;
833        let connect = self.connector.connect(&self.host, self.port);
834        let mut framed = match hold(&mut self.commands, &mut self.replays, reason, connect).await? {
835            Ok(stream) => Framed::new(stream),
836            Err(reason) => return Ok(Err(reason)),
837        };
838        let mut sent = Replays::new();
839        loop {
840            let next = self
841                .replays
842                .iter()
843                .find(|(key, stored)| sent.get(*key) != Some(*stored))
844                .map(|(key, stored)| (key.clone(), stored.clone()));
845            let Some((key, stored)) = next else {
846                return Ok(Ok((framed, sent.len())));
847            };
848            sent.insert(key.clone(), stored.clone());
849            let id = self.fresh_id();
850            let mut pushed = Vec::new();
851            let step = Box::pin(replay(
852                &mut framed,
853                id,
854                stored.clone(),
855                limit,
856                &self.events,
857                &mut pushed,
858            ));
859            let answer = match hold(&mut self.commands, &mut self.replays, reason, step).await? {
860                Ok(answer) => answer,
861                Err(reason) => return Ok(Err(reason)),
862            };
863            for session in pushed {
864                self.end(session);
865            }
866            match answer {
867                Ok(answer) => self.ended.granted(&answer),
868                Err(error) => {
869                    log_debug!(%error, "replayed request failed");
870                    let refused = token_refused(&key, &stored, &error);
871                    let _ = self.events.send(Event::ReplayFailed { key, error });
872                    if let Some(session) = refused {
873                        self.end(session);
874                    }
875                }
876            }
877            hold(
878                &mut self.commands,
879                &mut self.replays,
880                reason,
881                sleep(spacing),
882            )
883            .await?;
884        }
885    }
886
887    async fn serve(&mut self, framed: &mut Framed<C::Stream>) -> Exit {
888        loop {
889            if let Err(reason) = self.flush(framed).await {
890                return Exit::Lost(reason);
891            }
892            let wake = self.next_wake();
893            let capacity = self.options.command_capacity;
894            tokio::select! {
895                incoming = framed.receive() => match incoming {
896                    Ok(message) => self.heard(message),
897                    Err(reason) => return Exit::Lost(reason),
898                },
899                command = self.commands.recv(), if self.queue.len() < capacity => match command {
900                    Some(Command::Request(outbound)) => self.queue.push_back(outbound),
901                    Some(Command::Replay(key, stored)) => store(&mut self.replays, key, stored),
902                    Some(Command::Reconnect) => return Exit::Lost(TransportError::Closed),
903                    Some(Command::Shutdown) | None => return Exit::Shutdown,
904                },
905                () = reached(wake) => {
906                    if let Err(reason) = self.tick(framed).await {
907                        return Exit::Lost(reason);
908                    }
909                }
910            }
911        }
912    }
913
914    fn end(&mut self, session: SessionEnded) {
915        log_debug!(%session, "authorisation ended");
916        if let Some(account) = session.account {
917            self.replays.retain(|_, stored| {
918                account_of(stored.payload_type, &stored.payload) != Some(account)
919            });
920            self.ended.accounts.insert(account, session.cause.clone());
921        } else {
922            for stored in self.replays.values() {
923                if let Some(account) = account_of(stored.payload_type, &stored.payload) {
924                    self.ended.accounts.insert(account, session.cause.clone());
925                }
926            }
927            self.replays.clear();
928            self.ended.application = Some(session.cause.clone());
929        }
930        let _ = self.events.send(Event::SessionEnded(session));
931    }
932
933    fn fresh_id(&mut self) -> String {
934        self.next_id = self.next_id.saturating_add(1);
935        self.next_id.to_string()
936    }
937
938    fn heard(&mut self, message: ProtoMessage) {
939        self.silent_at = within(Instant::now(), self.options.silence());
940        self.dispatch(message);
941    }
942
943    fn next_wake(&self) -> Option<Instant> {
944        let deadline = self.deadlines.front().and_then(|(deadline, _)| *deadline);
945        [self.heartbeat_at, self.silent_at, deadline, self.send_at]
946            .into_iter()
947            .flatten()
948            .min()
949    }
950
951    fn next_ready(&mut self, now: Instant) -> Option<usize> {
952        self.send_at = None;
953        if let Some(until) = self.paused_until {
954            if until > now {
955                if !self.queue.is_empty() {
956                    self.send_at = Some(until);
957                }
958                return None;
959            }
960            self.paused_until = None;
961        }
962        let limits = [self.options.request_limit, self.options.historical_limit];
963        let mut ready = [None, None];
964        for ((throttle, limit), ready) in self.throttles.iter_mut().zip(limits).zip(&mut ready) {
965            *ready = throttle.ready_at(now, limit);
966        }
967        for (index, outbound) in self.queue.iter().enumerate() {
968            match ready
969                .get(usize::from(outbound.historical))
970                .copied()
971                .flatten()
972            {
973                None => return Some(index),
974                Some(at) => self.send_at = Some(self.send_at.map_or(at, |known| known.min(at))),
975            }
976        }
977        None
978    }
979
980    async fn flush(&mut self, framed: &mut Framed<C::Stream>) -> Result<(), TransportError> {
981        loop {
982            let now = Instant::now();
983            let Some(outbound) = self
984                .next_ready(now)
985                .and_then(|index| self.queue.remove(index))
986            else {
987                return Ok(());
988            };
989            if outbound.reply.is_closed() {
990                continue;
991            }
992            if let Some(ended) = self.ended.refusal(outbound.payload_type, &outbound.payload) {
993                let _ = outbound.reply.send(Err(ended.into()));
994                continue;
995            }
996            let id = self.fresh_id();
997            let message = ProtoMessage {
998                payload_type: outbound.payload_type,
999                payload: Some(outbound.payload.clone()),
1000                client_msg_id: Some(id.clone()),
1001            };
1002            match write(framed, &message, self.options.timeout).await {
1003                Ok(()) => {
1004                    bump(&self.counters.requests_sent);
1005                    if let Some(throttle) = self.throttles.get_mut(usize::from(outbound.historical))
1006                    {
1007                        throttle.sent.push_back(now);
1008                    }
1009                    self.heartbeat_at = within(now, self.options.heartbeat_interval);
1010                    self.deadlines
1011                        .push_back((within(now, self.options.timeout), id.clone()));
1012                    self.pending.insert(id, outbound);
1013                }
1014                Err(error @ TransportError::FrameTooLong { .. }) => {
1015                    let _ = outbound.reply.send(Err(error.into()));
1016                }
1017                Err(reason) => {
1018                    let error = outbound.interrupted(Interruption::Transport(reason.clone()));
1019                    let _ = outbound.reply.send(Err(error));
1020                    return Err(reason);
1021                }
1022            }
1023        }
1024    }
1025
1026    async fn tick(&mut self, framed: &mut Framed<C::Stream>) -> Result<(), TransportError> {
1027        let now = Instant::now();
1028        while let Some((deadline, id)) = self.deadlines.pop_front() {
1029            if deadline.is_none_or(|deadline| deadline > now) {
1030                self.deadlines.push_front((deadline, id));
1031                break;
1032            }
1033            if let Some(outbound) = self.pending.remove(&id) {
1034                let error = outbound.interrupted(Interruption::Timeout(self.options.timeout));
1035                let _ = outbound.reply.send(Err(error));
1036            }
1037        }
1038        if self.silent_at.is_some_and(|silent_at| now >= silent_at) {
1039            let silence = self.options.silence();
1040            let late = timeout(Duration::ZERO, framed.receive())
1041                .await
1042                .unwrap_or(Err(TransportError::Idle(silence)));
1043            self.heard(late?);
1044        }
1045        if self
1046            .heartbeat_at
1047            .is_some_and(|heartbeat_at| now >= heartbeat_at)
1048        {
1049            let beat = ProtoHeartbeatEvent::default().to_message(None);
1050            write(framed, &beat, self.options.timeout).await?;
1051            bump(&self.counters.heartbeats_sent);
1052            self.heartbeat_at = within(Instant::now(), self.options.heartbeat_interval);
1053        }
1054        Ok(())
1055    }
1056
1057    fn dispatch(&mut self, message: ProtoMessage) {
1058        if message.payload_type == ProtoHeartbeatEvent::PAYLOAD_TYPE {
1059            bump(&self.counters.heartbeats_received);
1060            return;
1061        }
1062        let waiting = message
1063            .client_msg_id
1064            .as_ref()
1065            .and_then(|id| self.pending.remove(id));
1066        let Some(mut outbound) = waiting else {
1067            let sessions = sessions_ended_by(&message);
1068            let _ = self.events.send(Event::Message(message));
1069            for session in sessions {
1070                self.end(session);
1071            }
1072            return;
1073        };
1074        self.ended.granted(&message);
1075        if outbound.once {
1076            let _ = self.events.send(Event::Message(message.clone()));
1077        }
1078        let result = outcome(message);
1079        if !outbound.once && over_limit(&result) && outbound.attempts < self.options.retries {
1080            let pause = self
1081                .options
1082                .backoff
1083                .saturating_mul(1 << outbound.attempts.min(16))
1084                .min(BACKOFF_LIMIT.max(self.options.backoff));
1085            let until = after(Instant::now(), pause);
1086            log_debug!(
1087                payload_type = outbound.payload_type,
1088                ?pause,
1089                "limit exceeded"
1090            );
1091            self.paused_until = Some(self.paused_until.map_or(until, |known| known.max(until)));
1092            outbound.attempts += 1;
1093            bump(&self.counters.throttle_retries);
1094            self.queue.push_front(outbound);
1095        } else {
1096            let _ = outbound.reply.send(result);
1097        }
1098    }
1099
1100    fn fail_in_flight(&mut self, reason: &TransportError) {
1101        self.deadlines.clear();
1102        for (_, outbound) in self.pending.drain() {
1103            let error = outbound.interrupted(Interruption::Transport(reason.clone()));
1104            let _ = outbound.reply.send(Err(error));
1105        }
1106        for outbound in self.queue.drain(..) {
1107            let _ = outbound.reply.send(Err(reason.clone().into()));
1108        }
1109        self.reset_pacing();
1110    }
1111
1112    fn close_in_flight(&mut self, error: &ConnectorError) {
1113        self.deadlines.clear();
1114        for (_, outbound) in self.pending.drain() {
1115            let _ = outbound.reply.send(Err(error.clone()));
1116        }
1117        for outbound in self.queue.drain(..) {
1118            let _ = outbound.reply.send(Err(error.clone()));
1119        }
1120        self.reset_pacing();
1121    }
1122
1123    fn reset_pacing(&mut self) {
1124        self.paused_until = None;
1125        self.send_at = None;
1126        for throttle in &mut self.throttles {
1127            throttle.sent.clear();
1128        }
1129    }
1130
1131    fn fail_all(&mut self, error: &ConnectorError) {
1132        self.close_in_flight(error);
1133        self.commands.close();
1134        while let Ok(command) = self.commands.try_recv() {
1135            if let Command::Request(outbound) = command {
1136                let _ = outbound.reply.send(Err(error.clone()));
1137            }
1138        }
1139    }
1140}
1141
1142#[derive(Default)]
1143struct Throttle {
1144    sent: VecDeque<Instant>,
1145}
1146
1147impl Throttle {
1148    fn ready_at(&mut self, now: Instant, limit: u32) -> Option<Instant> {
1149        while self
1150            .sent
1151            .front()
1152            .is_some_and(|sent| now.duration_since(*sent) >= LIMIT_WINDOW)
1153        {
1154            self.sent.pop_front();
1155        }
1156        let limit = usize::try_from(limit).unwrap_or(usize::MAX);
1157        let excess = (self.sent.len() + 1).checked_sub(limit)?;
1158        let blocking = excess.checked_sub(1)?;
1159        self.sent
1160            .get(blocking)
1161            .map(|sent| after(*sent, LIMIT_WINDOW))
1162    }
1163}
1164
1165fn after(start: Instant, period: Duration) -> Instant {
1166    start.checked_add(period.min(LONGEST_WAIT)).unwrap_or(start)
1167}
1168
1169fn within(start: Instant, period: Duration) -> Option<Instant> {
1170    if period > LONGEST_WAIT {
1171        return None;
1172    }
1173    start.checked_add(period)
1174}
1175
1176async fn reached(wake: Option<Instant>) {
1177    match wake {
1178        Some(wake) => sleep_until(wake).await,
1179        None => pending().await,
1180    }
1181}
1182
1183fn over_limit(result: &Result<ProtoMessage, ConnectorError>) -> bool {
1184    matches!(
1185        result,
1186        Err(ConnectorError::Server(error)) if matches!(
1187            error.code,
1188            ErrorCode::RequestFrequencyExceeded | ErrorCode::BlockedPayloadType
1189        )
1190    )
1191}
1192
1193fn store(replays: &mut Replays, key: ReplayKey, stored: Option<Stored>) {
1194    match stored {
1195        Some(stored) => {
1196            replays.insert(key, stored);
1197        }
1198        None => {
1199            replays.remove(&key);
1200        }
1201    }
1202}
1203
1204async fn write<S: AsyncRead + AsyncWrite + Unpin>(
1205    framed: &mut Framed<S>,
1206    message: &ProtoMessage,
1207    limit: Duration,
1208) -> Result<(), TransportError> {
1209    timeout(limit, framed.send(message))
1210        .await
1211        .unwrap_or(Err(TransportError::WriteTimeout(limit)))
1212}
1213
1214async fn close<S: AsyncRead + AsyncWrite + Unpin>(framed: &mut Framed<S>, limit: Duration) {
1215    let _ = timeout(limit, framed.close()).await;
1216}
1217
1218async fn hold<F: Future>(
1219    commands: &mut mpsc::Receiver<Command>,
1220    replays: &mut Replays,
1221    reason: &TransportError,
1222    work: F,
1223) -> Result<F::Output, ConnectorError> {
1224    tokio::pin!(work);
1225    loop {
1226        tokio::select! {
1227            output = &mut work => return Ok(output),
1228            command = commands.recv() => match command {
1229                Some(Command::Request(outbound)) => {
1230                    let _ = outbound.reply.send(Err(reason.clone().into()));
1231                }
1232                Some(Command::Replay(key, stored)) => store(replays, key, stored),
1233                Some(Command::Reconnect) => {}
1234                Some(Command::Shutdown) | None => return Err(ConnectorError::Closed),
1235            },
1236        }
1237    }
1238}
1239
1240async fn replay<S: AsyncRead + AsyncWrite + Unpin>(
1241    framed: &mut Framed<S>,
1242    id: String,
1243    stored: Stored,
1244    limit: Duration,
1245    events: &broadcast::Sender<Event>,
1246    pushed: &mut Vec<SessionEnded>,
1247) -> Result<Result<ProtoMessage, ConnectorError>, TransportError> {
1248    let message = ProtoMessage {
1249        payload_type: stored.payload_type,
1250        payload: Some(stored.payload),
1251        client_msg_id: Some(id.clone()),
1252    };
1253    write(framed, &message, limit).await?;
1254    Ok(
1255        match timeout(limit, answer_to(framed, &id, events, pushed)).await {
1256            Ok(answer) => outcome(answer?),
1257            Err(_) => Err(ConnectorError::Timeout(limit)),
1258        },
1259    )
1260}
1261
1262async fn answer_to<S: AsyncRead + AsyncWrite + Unpin>(
1263    framed: &mut Framed<S>,
1264    id: &str,
1265    events: &broadcast::Sender<Event>,
1266    pushed: &mut Vec<SessionEnded>,
1267) -> Result<ProtoMessage, TransportError> {
1268    loop {
1269        let message = framed.receive().await?;
1270        if message.client_msg_id.as_deref() == Some(id) {
1271            return Ok(message);
1272        }
1273        if message.payload_type != ProtoHeartbeatEvent::PAYLOAD_TYPE {
1274            pushed.extend(sessions_ended_by(&message));
1275            let _ = events.send(Event::Message(message));
1276        }
1277    }
1278}
1279
1280fn outcome(message: ProtoMessage) -> Result<ProtoMessage, ConnectorError> {
1281    if message.payload_type == ProtoOaErrorRes::PAYLOAD_TYPE {
1282        Err(ProtoOaErrorRes::from_message(&message)
1283            .map_or_else(|error| error, |response| ServerError::from(response).into()))
1284    } else if message.payload_type == ProtoErrorRes::PAYLOAD_TYPE {
1285        Err(ProtoErrorRes::from_message(&message)
1286            .map_or_else(|error| error, |response| ServerError::from(response).into()))
1287    } else {
1288        Ok(message)
1289    }
1290}
1291
1292#[cfg(test)]
1293mod tests {
1294    use std::time::Duration;
1295
1296    use super::{
1297        ConnectionOptions, DEFAULT_INBOUND_TIMEOUT, Event, LIMIT_WINDOW, LONGEST_WAIT,
1298        MAX_COMMAND_CAPACITY, MAX_EVENT_CAPACITY, ReplayKey, ReplayStage,
1299        SERVER_HEARTBEAT_INTERVAL, Throttle, after, outcome, over_limit, within,
1300    };
1301    use crate::common::config::ConfigurationOpenApi;
1302    use crate::common::errors::{ConnectorError, ErrorCode, TransportError};
1303    use crate::open_api::messages::{
1304        ProtoErrorRes, ProtoMessage, ProtoOaErrorRes, ProtoOaSpotEvent, ProtoOaVersionRes,
1305    };
1306    use crate::open_api::payload::Payload;
1307    use tokio::time::Instant;
1308
1309    #[test]
1310    fn options_take_timing_and_retries_of_the_configuration() {
1311        let configuration = ConfigurationOpenApi::builder()
1312            .client_id("id")
1313            .client_secret("secret")
1314            .access_token("token")
1315            .timeout(Duration::from_secs(3))
1316            .reconnect_delay(Duration::from_secs(2))
1317            .retries(7)
1318            .backoff(Duration::from_millis(250))
1319            .build()
1320            .expect("configuration");
1321        let options = ConnectionOptions::from(&configuration);
1322        assert_eq!(
1323            options,
1324            ConnectionOptions::new()
1325                .timeout(Duration::from_secs(3))
1326                .reconnect_delay(Duration::from_secs(2))
1327                .retries(7)
1328                .backoff(Duration::from_millis(250))
1329        );
1330    }
1331
1332    #[test]
1333    fn silence_allowed_follows_the_pace_of_the_server_unless_it_is_set() {
1334        assert_eq!(DEFAULT_INBOUND_TIMEOUT, Duration::from_secs(75));
1335        assert!(DEFAULT_INBOUND_TIMEOUT >= SERVER_HEARTBEAT_INTERVAL * 2);
1336        assert_eq!(ConnectionOptions::new().silence(), DEFAULT_INBOUND_TIMEOUT);
1337        let quick = ConnectionOptions::new()
1338            .heartbeat_interval(Duration::from_secs(2))
1339            .timeout(Duration::from_secs(1));
1340        assert_eq!(quick.silence(), DEFAULT_INBOUND_TIMEOUT);
1341        assert_eq!(
1342            quick.clone().inbound_timeout(Duration::ZERO).silence(),
1343            Duration::from_millis(1)
1344        );
1345        assert_eq!(
1346            quick.inbound_timeout(Duration::from_secs(90)).silence(),
1347            Duration::from_secs(90)
1348        );
1349    }
1350
1351    #[test]
1352    fn options_raise_degenerate_values() {
1353        let options = ConnectionOptions::new()
1354            .timeout(Duration::ZERO)
1355            .heartbeat_interval(Duration::ZERO)
1356            .command_capacity(0)
1357            .event_capacity(0)
1358            .reconnect_delay(Duration::ZERO)
1359            .reconnect_attempts(Some(4))
1360            .request_limit(0)
1361            .historical_limit(0)
1362            .retries(0)
1363            .backoff(Duration::ZERO);
1364        assert_eq!(options.timeout, Duration::from_millis(1));
1365        assert_eq!(options.heartbeat_interval, Duration::from_millis(1));
1366        assert_eq!(options.command_capacity, 1);
1367        assert_eq!(options.event_capacity, 1);
1368        let largest = ConnectionOptions::new()
1369            .command_capacity(usize::MAX)
1370            .event_capacity(usize::MAX);
1371        assert_eq!(largest.command_capacity, MAX_COMMAND_CAPACITY);
1372        assert_eq!(largest.event_capacity, MAX_EVENT_CAPACITY);
1373        assert_eq!(options.reconnect_delay, Duration::from_millis(1));
1374        assert_eq!(options.reconnect_attempts, Some(4));
1375        assert_eq!(options.request_limit, 1);
1376        assert_eq!(options.historical_limit, 1);
1377        assert_eq!(options.retries, 0);
1378        assert_eq!(options.backoff, Duration::from_millis(1));
1379        assert_eq!(ConnectionOptions::new().reconnect_attempts, None);
1380    }
1381
1382    #[test]
1383    fn error_responses_become_server_errors() {
1384        let open_api = ProtoOaErrorRes {
1385            error_code: "CH_CLIENT_AUTH_FAILURE".to_owned(),
1386            ..ProtoOaErrorRes::default()
1387        };
1388        assert!(matches!(
1389            outcome(open_api.to_message(None)),
1390            Err(ConnectorError::Server(error)) if error.code == ErrorCode::ChClientAuthFailure
1391        ));
1392        let common = ProtoErrorRes {
1393            error_code: "UNSUPPORTED_MESSAGE".to_owned(),
1394            ..ProtoErrorRes::default()
1395        };
1396        assert!(matches!(
1397            outcome(common.to_message(None)),
1398            Err(ConnectorError::Server(error)) if error.code == ErrorCode::UnsupportedMessage
1399        ));
1400        for payload_type in [50, 2142] {
1401            let malformed = ProtoMessage {
1402                payload_type,
1403                payload: Some(vec![0xff]),
1404                client_msg_id: None,
1405            };
1406            assert!(matches!(
1407                outcome(malformed),
1408                Err(ConnectorError::Transport(TransportError::Decode(_)))
1409            ));
1410        }
1411        let fine = ProtoOaVersionRes::default().to_message(None);
1412        assert_eq!(outcome(fine.clone()), Ok(fine));
1413    }
1414
1415    #[test]
1416    fn event_decodes_only_the_message_it_carries() {
1417        let spot = ProtoOaSpotEvent {
1418            ctid_trader_account_id: 1,
1419            symbol_id: 2,
1420            ..ProtoOaSpotEvent::default()
1421        };
1422        let event = Event::Message(spot.to_message(None));
1423        assert_eq!(event.decode::<ProtoOaSpotEvent>(), Some(spot));
1424        assert_eq!(event.decode::<ProtoOaVersionRes>(), None);
1425        let lost = Event::Disconnected(TransportError::Closed);
1426        assert_eq!(lost.decode::<ProtoOaSpotEvent>(), None);
1427        let again = Event::Reconnected { attempt: 1 };
1428        assert_eq!(again.decode::<ProtoOaSpotEvent>(), None);
1429    }
1430
1431    #[test]
1432    fn replay_keys_order_by_stage_then_name() {
1433        let mut keys = [
1434            ReplayKey::new(ReplayStage::Subscription, "a"),
1435            ReplayKey::new(ReplayStage::Account, "2"),
1436            ReplayKey::new(ReplayStage::Application, "z"),
1437            ReplayKey::new(ReplayStage::Account, "1"),
1438        ];
1439        keys.sort();
1440        let order: Vec<(ReplayStage, &str)> =
1441            keys.iter().map(|key| (key.stage(), key.id())).collect();
1442        assert_eq!(
1443            order,
1444            vec![
1445                (ReplayStage::Application, "z"),
1446                (ReplayStage::Account, "1"),
1447                (ReplayStage::Account, "2"),
1448                (ReplayStage::Subscription, "a"),
1449            ]
1450        );
1451    }
1452
1453    #[test]
1454    fn throttle_names_the_moment_the_next_request_may_go() {
1455        let start = Instant::now();
1456        let mut throttle = Throttle::default();
1457        assert_eq!(throttle.ready_at(start, 2), None);
1458        throttle.sent.push_back(start);
1459        assert_eq!(throttle.ready_at(start, 2), None);
1460        throttle.sent.push_back(start + Duration::from_millis(10));
1461        assert_eq!(throttle.ready_at(start, 2), Some(start + LIMIT_WINDOW));
1462        throttle.sent.push_back(start + Duration::from_millis(20));
1463        assert_eq!(
1464            throttle.ready_at(start, 2),
1465            Some(start + Duration::from_millis(10) + LIMIT_WINDOW)
1466        );
1467        let later = start + LIMIT_WINDOW + Duration::from_millis(10);
1468        assert_eq!(throttle.ready_at(later, 2), None);
1469        assert_eq!(throttle.sent.len(), 1);
1470    }
1471
1472    #[test]
1473    fn waits_too_long_for_the_clock_end_at_the_longest_wait() {
1474        let start = Instant::now();
1475        let brief = Duration::from_secs(90);
1476        assert_eq!(after(start, brief), start + brief);
1477        assert_eq!(after(start, LONGEST_WAIT), start + LONGEST_WAIT);
1478        assert_eq!(after(start, Duration::MAX), start + LONGEST_WAIT);
1479        let far = after(start, Duration::MAX);
1480        assert_eq!(after(far, Duration::MAX), far + LONGEST_WAIT);
1481        assert_eq!(within(start, brief), Some(start + brief));
1482        assert_eq!(within(start, LONGEST_WAIT), Some(start + LONGEST_WAIT));
1483        assert_eq!(within(start, LONGEST_WAIT + Duration::from_nanos(1)), None);
1484        assert_eq!(within(start, Duration::MAX), None);
1485        let options = ConnectionOptions::new()
1486            .timeout(Duration::MAX)
1487            .heartbeat_interval(Duration::MAX);
1488        assert_eq!(options.silence(), DEFAULT_INBOUND_TIMEOUT);
1489        assert_eq!(
1490            options.inbound_timeout(Duration::MAX).silence(),
1491            Duration::MAX
1492        );
1493        let mut throttle = Throttle::default();
1494        throttle.sent.push_back(far);
1495        assert_eq!(throttle.ready_at(far, 1), Some(far + LIMIT_WINDOW));
1496    }
1497
1498    #[test]
1499    fn only_limit_errors_are_retried() {
1500        let refusal = |code: &str| {
1501            outcome(
1502                ProtoOaErrorRes {
1503                    error_code: code.to_owned(),
1504                    ..ProtoOaErrorRes::default()
1505                }
1506                .to_message(None),
1507            )
1508        };
1509        assert!(over_limit(&refusal("REQUEST_FREQUENCY_EXCEEDED")));
1510        assert!(over_limit(&refusal("BLOCKED_PAYLOAD_TYPE")));
1511        assert!(!over_limit(&refusal("INVALID_REQUEST")));
1512        assert!(!over_limit(&Ok(
1513            ProtoOaVersionRes::default().to_message(None)
1514        )));
1515        assert!(!over_limit(&Err(ConnectorError::Closed)));
1516    }
1517}