Skip to main content

open_net/api/ws/
application_heartbeat.rs

1//! Application-level WebSocket heartbeat support.
2//!
3//! WebSocket control Ping/Pong frames are handled by the transport heartbeat.
4//! This module provides a separate, bounded loop for exchanges that require a
5//! text or binary heartbeat message at the application protocol level.
6
7use super::{
8    DisconnectedPolicy, IncomingMessage, Message, MessageReceiver, ReceiveOptions, ReceiveOverflow,
9    SendOptions, Sender,
10};
11use crate::error::{ErrorKind, ErrorStage, ReceiveError};
12use crate::module::ws_client::session_runtime::SessionRuntime;
13use crate::{NetError, Result};
14use std::panic::{catch_unwind, AssertUnwindSafe};
15use std::sync::{Arc, Mutex, MutexGuard};
16use std::time::{Duration, Instant as StdInstant, SystemTime};
17use tokio::task::JoinHandle;
18use tokio::time::{self, Instant};
19use tokio_util::sync::CancellationToken;
20
21/// Callback used to recognize an application-level heartbeat response.
22///
23/// The callback must only inspect the immutable message. A callback panic is
24/// converted to [`ErrorKind::CallbackPanicked`] and never escapes the task.
25/// Matching is message-level and connection-scoped; protocols that can deliver
26/// delayed or repeated responses should include a request nonce in the payload
27/// and verify it in this callback.
28pub type ApplicationHeartbeatMatcher =
29    Arc<dyn Fn(&IncomingMessage) -> bool + Send + Sync + 'static>;
30
31/// Action to take after an application heartbeat send or response failure.
32#[derive(Clone, Copy, Debug, Eq, PartialEq)]
33pub enum ApplicationHeartbeatFailureAction {
34    /// Record the failure and continue with the next scheduled heartbeat.
35    Continue,
36    /// Record the failure and stop the application heartbeat loop.
37    Stop,
38}
39
40/// Configuration for a fixed-interval application heartbeat loop.
41#[derive(Clone)]
42pub struct ApplicationHeartbeatConfig {
43    /// Delay between heartbeat attempts. The first heartbeat is sent after this delay.
44    pub interval: Duration,
45    /// Text or binary payload sent on every heartbeat attempt.
46    pub payload: Message,
47    /// Maximum time allowed for queue admission and the local write.
48    pub send_timeout: Duration,
49    /// How long to wait for a matching response after a successful write.
50    /// `None` disables response matching and timeout accounting.
51    pub response_timeout: Option<Duration>,
52    /// Number of consecutive missed responses tolerated before `Stop` takes effect.
53    pub max_missed_responses: u32,
54    /// Optional application response matcher. When absent, writes are observed only.
55    pub matcher: Option<ApplicationHeartbeatMatcher>,
56    /// Receive queue used exclusively by the heartbeat loop when a matcher is set.
57    /// The default is [`ReceiveOverflow::DropOldest`] so unrelated business
58    /// traffic cannot close the session; selecting `Disconnect` makes queue
59    /// overflow a heartbeat receiver failure by design.
60    pub receive_options: ReceiveOptions,
61    /// Failure handling policy for writes, receiver failures, and response timeouts.
62    pub failure_action: ApplicationHeartbeatFailureAction,
63}
64
65impl ApplicationHeartbeatConfig {
66    /// Creates a fixed-interval heartbeat configuration with no response matching.
67    pub fn new(interval: Duration, payload: impl Into<Message>) -> Self {
68        Self {
69            interval,
70            payload: payload.into(),
71            send_timeout: Duration::from_secs(10),
72            response_timeout: None,
73            max_missed_responses: 1,
74            matcher: None,
75            receive_options: ReceiveOptions {
76                overflow: ReceiveOverflow::DropOldest,
77                ..ReceiveOptions::default()
78            },
79            failure_action: ApplicationHeartbeatFailureAction::Continue,
80        }
81    }
82
83    /// Sets the timeout used for queue admission and local WebSocket writing.
84    pub fn with_send_timeout(mut self, timeout: Duration) -> Self {
85        self.send_timeout = timeout;
86        self
87    }
88
89    /// Enables response matching with the supplied timeout and matcher.
90    pub fn with_response_matcher(
91        mut self,
92        timeout: Duration,
93        matcher: impl Fn(&IncomingMessage) -> bool + Send + Sync + 'static,
94    ) -> Self {
95        self.response_timeout = Some(timeout);
96        self.matcher = Some(Arc::new(matcher));
97        self
98    }
99
100    /// Sets the maximum number of consecutive response timeouts.
101    pub fn with_max_missed_responses(mut self, max_missed_responses: u32) -> Self {
102        self.max_missed_responses = max_missed_responses;
103        self
104    }
105
106    /// Sets the receive queue options used by the heartbeat loop.
107    pub fn with_receive_options(mut self, receive_options: ReceiveOptions) -> Self {
108        self.receive_options = receive_options;
109        self
110    }
111
112    /// Sets whether failures stop or leave the loop running.
113    pub fn with_failure_action(
114        mut self,
115        failure_action: ApplicationHeartbeatFailureAction,
116    ) -> Self {
117        self.failure_action = failure_action;
118        self
119    }
120
121    /// Validates values that would otherwise create a busy loop or an unbounded wait.
122    pub fn validate(&self) -> Result<()> {
123        if self.interval.is_zero() {
124            return Err(NetError::config(
125                "application_heartbeat.interval",
126                "must be greater than zero",
127            ));
128        }
129        if Instant::now().checked_add(self.interval).is_none() {
130            return Err(NetError::config(
131                "application_heartbeat.interval",
132                "cannot be represented as a timer deadline",
133            ));
134        }
135        if self.send_timeout.is_zero() {
136            return Err(NetError::config(
137                "application_heartbeat.send_timeout",
138                "must be greater than zero",
139            ));
140        }
141        if StdInstant::now().checked_add(self.send_timeout).is_none() {
142            return Err(NetError::config(
143                "application_heartbeat.send_timeout",
144                "cannot be represented as a deadline",
145            ));
146        }
147        if self.max_missed_responses == 0 {
148            return Err(NetError::config(
149                "application_heartbeat.max_missed_responses",
150                "must be greater than zero",
151            ));
152        }
153        if self.matcher.is_some() && self.receive_options.include_control_frames {
154            return Err(NetError::config(
155                "application_heartbeat.receive_options.include_control_frames",
156                "must be false so control Pong frames cannot acknowledge application heartbeats",
157            ));
158        }
159        if self.matcher.is_some() && self.response_timeout.is_none() {
160            return Err(NetError::config(
161                "application_heartbeat.response_timeout",
162                "is required when matcher is set",
163            ));
164        }
165        if let Some(timeout) = self.response_timeout {
166            if StdInstant::now().checked_add(timeout).is_none() {
167                return Err(NetError::config(
168                    "application_heartbeat.response_timeout",
169                    "must fit an absolute deadline",
170                ));
171            }
172            if timeout.is_zero() {
173                return Err(NetError::config(
174                    "application_heartbeat.response_timeout",
175                    "must be greater than zero",
176                ));
177            }
178            if Instant::now().checked_add(timeout).is_none() {
179                return Err(NetError::config(
180                    "application_heartbeat.response_timeout",
181                    "cannot be represented as a deadline",
182                ));
183            }
184            if self.matcher.is_none() {
185                return Err(NetError::config(
186                    "application_heartbeat.matcher",
187                    "is required when response_timeout is set",
188                ));
189            }
190        }
191        if self.matcher.is_some() {
192            self.receive_options.validate()?;
193        }
194        Ok(())
195    }
196}
197
198impl std::fmt::Debug for ApplicationHeartbeatConfig {
199    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
200        formatter
201            .debug_struct("ApplicationHeartbeatConfig")
202            .field("interval", &self.interval)
203            .field("payload_bytes", &self.payload.len())
204            .field("send_timeout", &self.send_timeout)
205            .field("response_timeout", &self.response_timeout)
206            .field("max_missed_responses", &self.max_missed_responses)
207            .field("has_matcher", &self.matcher.is_some())
208            .field("receive_options", &self.receive_options)
209            .field("failure_action", &self.failure_action)
210            .finish()
211    }
212}
213
214/// Current state of an application heartbeat loop.
215#[derive(Clone, Copy, Debug, Eq, PartialEq)]
216pub enum ApplicationHeartbeatState {
217    /// The loop is waiting for its next scheduled send.
218    Running,
219    /// A heartbeat was written and a response is being matched.
220    WaitingForResponse,
221    /// The loop has stopped because its handle was stopped, its session ended, or policy stopped it.
222    Stopped,
223}
224
225/// Immutable counters and last-error classification for an application heartbeat.
226#[derive(Clone, Copy, Debug, Eq, PartialEq)]
227pub struct ApplicationHeartbeatSnapshot {
228    /// Current loop state.
229    pub state: ApplicationHeartbeatState,
230    /// Number of successful local writes.
231    pub sent: u64,
232    /// Number of matched application responses.
233    pub matched_responses: u64,
234    /// Number of response deadlines reached without a match.
235    pub missed_responses: u64,
236    /// Number of send, receive, matcher, or response failures.
237    pub failures: u64,
238    /// Classification of the most recent failure, if any.
239    pub last_error: Option<ErrorKind>,
240}
241
242impl Default for ApplicationHeartbeatSnapshot {
243    fn default() -> Self {
244        Self {
245            state: ApplicationHeartbeatState::Running,
246            sent: 0,
247            matched_responses: 0,
248            missed_responses: 0,
249            failures: 0,
250            last_error: None,
251        }
252    }
253}
254
255/// Controls one application heartbeat loop.
256///
257/// Dropping the handle requests cancellation and aborts only the heartbeat task;
258/// the owning WebSocket session remains alive.
259pub struct ApplicationHeartbeatHandle {
260    cancellation: CancellationToken,
261    snapshot: Arc<Mutex<ApplicationHeartbeatSnapshot>>,
262    task: Option<JoinHandle<()>>,
263}
264
265impl ApplicationHeartbeatHandle {
266    pub(crate) fn new(
267        sender: Sender,
268        receiver: Option<MessageReceiver>,
269        config: ApplicationHeartbeatConfig,
270    ) -> Result<Self> {
271        config.validate()?;
272        let cancellation = CancellationToken::new();
273        let snapshot = Arc::new(Mutex::new(ApplicationHeartbeatSnapshot::default()));
274        let runtime = sender.runtime.clone();
275        let task = runtime.clone().spawn(run_heartbeat(
276            sender,
277            receiver,
278            config,
279            cancellation.clone(),
280            snapshot.clone(),
281            runtime,
282        ))?;
283        Ok(Self {
284            cancellation,
285            snapshot,
286            task: Some(task),
287        })
288    }
289
290    /// Requests the loop to stop. The request is idempotent.
291    pub fn stop(&self) {
292        self.cancellation.cancel();
293    }
294
295    /// Returns a point-in-time snapshot without waiting for the loop.
296    pub fn snapshot(&self) -> ApplicationHeartbeatSnapshot {
297        snapshot_copy(&self.snapshot)
298    }
299
300    /// Waits for the loop task to finish and returns its final snapshot.
301    pub async fn stopped(mut self) -> Result<ApplicationHeartbeatSnapshot> {
302        let task = self
303            .task
304            .take()
305            .ok_or_else(|| NetError::from(ErrorKind::Internal).with_stage(ErrorStage::Runtime))?;
306        task.await.map_err(|error| {
307            NetError::with_source(ErrorKind::EngineDropped, error).with_stage(ErrorStage::Runtime)
308        })?;
309        Ok(snapshot_copy(&self.snapshot))
310    }
311}
312
313impl Drop for ApplicationHeartbeatHandle {
314    fn drop(&mut self) {
315        self.cancellation.cancel();
316        if let Some(task) = self.task.take() {
317            task.abort();
318        }
319    }
320}
321
322#[derive(Clone, Copy, Debug, Eq, PartialEq)]
323enum ResponseWait {
324    Matched,
325    TimedOut,
326    Closed,
327    Cancelled,
328    Failed(ErrorKind),
329}
330
331async fn run_heartbeat(
332    sender: Sender,
333    mut receiver: Option<MessageReceiver>,
334    config: ApplicationHeartbeatConfig,
335    cancellation: CancellationToken,
336    snapshot: Arc<Mutex<ApplicationHeartbeatSnapshot>>,
337    runtime: Arc<SessionRuntime>,
338) {
339    let now = Instant::now();
340    let first_tick = match now.checked_add(config.interval) {
341        Some(deadline) => deadline,
342        None => now,
343    };
344    let mut ticker = time::interval_at(first_tick, config.interval);
345    ticker.set_missed_tick_behavior(time::MissedTickBehavior::Skip);
346    let mut consecutive_missed = 0_u32;
347    let session_closed = runtime.closed_token();
348
349    'heartbeat: loop {
350        tokio::select! {
351            _ = cancellation.cancelled() => break 'heartbeat,
352            _ = session_closed.cancelled() => break 'heartbeat,
353            _ = ticker.tick() => {
354                snapshot_update(&snapshot, |current| {
355                    current.state = ApplicationHeartbeatState::Running;
356                });
357                let options = match send_options(config.send_timeout) {
358                    Ok(options) => options,
359                    Err(error) => {
360                        // A configuration/deadline failure is not a missed
361                        // response. Keep the response-timeout streak scoped to
362                        // actual response deadlines.
363                        consecutive_missed = 0;
364                        let should_stop = record_failure(
365                            &snapshot,
366                            &error,
367                            config.failure_action,
368                        );
369                        if should_stop {
370                            break 'heartbeat;
371                        }
372                        continue;
373                    }
374                };
375                let sent_at = SystemTime::now();
376                let send = sender
377                    .message(config.payload.clone())
378                    .options(options)
379                    .send();
380                let send_result = tokio::select! {
381                    _ = cancellation.cancelled() => break 'heartbeat,
382                    _ = session_closed.cancelled() => break 'heartbeat,
383                    result = send => result,
384                };
385                if let Err(error) = send_result {
386                    // A write failure breaks the response-timeout streak; the
387                    // next successful write starts a new response sequence.
388                    consecutive_missed = 0;
389                    let should_stop = record_failure(
390                        &snapshot,
391                        &error,
392                        config.failure_action,
393                    );
394                    if should_stop {
395                        break 'heartbeat;
396                    }
397                    continue;
398                }
399                snapshot_update(&snapshot, |current| {
400                    increment(&mut current.sent);
401                });
402                let (Some(matcher), Some(receiver)) =
403                    (config.matcher.as_ref(), receiver.as_mut())
404                else {
405                    continue;
406                };
407                let connection_id = match runtime.current_connection_id() {
408                    Ok(Some(connection_id)) => connection_id,
409                    Ok(None) => {
410                        // The write completed while the physical connection
411                        // was being retired or replaced. Do not turn this
412                        // lifecycle race into a response timeout for the new
413                        // connection; retry on the next interval instead.
414                        consecutive_missed = 0;
415                        let error = NetError::from(ErrorKind::NotConnected)
416                            .with_stage(ErrorStage::Runtime);
417                        let should_stop = record_failure(
418                            &snapshot,
419                            &error,
420                            config.failure_action,
421                        );
422                        if should_stop {
423                            break 'heartbeat;
424                        }
425                        continue;
426                    }
427                    Err(error) => {
428                        // A connection/lifecycle lookup failure is not a
429                        // response timeout and must not carry a stale streak
430                        // into the next heartbeat.
431                        consecutive_missed = 0;
432                        let should_stop = record_failure(
433                            &snapshot,
434                            &error,
435                            config.failure_action,
436                        );
437                        if should_stop {
438                            break 'heartbeat;
439                        }
440                        continue;
441                    }
442                };
443                snapshot_update(&snapshot, |current| {
444                    current.state = ApplicationHeartbeatState::WaitingForResponse;
445                });
446                let timeout = match config.response_timeout {
447                    Some(timeout) => timeout,
448                    None => continue,
449                };
450                match wait_for_response(
451                    receiver,
452                    matcher,
453                    timeout,
454                    &cancellation,
455                    &session_closed,
456                    Some(connection_id),
457                    sent_at,
458                ).await {
459                    ResponseWait::Matched => {
460                        consecutive_missed = 0;
461                        snapshot_update(&snapshot, |current| {
462                            increment(&mut current.matched_responses);
463                            current.state = ApplicationHeartbeatState::Running;
464                        });
465                    }
466                    ResponseWait::TimedOut => {
467                        consecutive_missed = consecutive_missed.saturating_add(1);
468                        snapshot_update(&snapshot, |current| {
469                            increment(&mut current.missed_responses);
470                            current.last_error = Some(ErrorKind::TimedOut);
471                            increment(&mut current.failures);
472                            current.state = ApplicationHeartbeatState::Running;
473                        });
474                        if config.failure_action == ApplicationHeartbeatFailureAction::Stop
475                            && consecutive_missed >= config.max_missed_responses
476                        {
477                            break 'heartbeat;
478                        }
479                    }
480                    ResponseWait::Closed | ResponseWait::Cancelled => break 'heartbeat,
481                    ResponseWait::Failed(kind) => {
482                        // Receiver and matcher failures are distinct from a
483                        // response deadline, so they reset consecutive misses.
484                        consecutive_missed = 0;
485                        let should_stop = record_failure_kind(
486                            &snapshot,
487                            kind,
488                            config.failure_action,
489                        );
490                        if should_stop {
491                            break 'heartbeat;
492                        }
493                    }
494                }
495            }
496        }
497    }
498    if let Some(receiver) = receiver.as_ref() {
499        receiver.unsubscribe();
500    }
501    snapshot_update(&snapshot, |current| {
502        current.state = ApplicationHeartbeatState::Stopped;
503    });
504}
505
506async fn wait_for_response(
507    receiver: &mut MessageReceiver,
508    matcher: &ApplicationHeartbeatMatcher,
509    timeout: Duration,
510    cancellation: &CancellationToken,
511    session_closed: &CancellationToken,
512    connection_id: Option<crate::ws::ConnectionId>,
513    sent_at: SystemTime,
514) -> ResponseWait {
515    let deadline = time::sleep(timeout);
516    tokio::pin!(deadline);
517    loop {
518        tokio::select! {
519            _ = cancellation.cancelled() => return ResponseWait::Cancelled,
520            _ = session_closed.cancelled() => return ResponseWait::Closed,
521            _ = &mut deadline => return ResponseWait::TimedOut,
522            result = receiver.recv() => {
523                match result {
524                    Ok(Some(message)) => {
525                        if connection_id != Some(message.connection_id()) {
526                            continue;
527                        }
528                        if message.received_at().duration_since(sent_at).is_err() {
529                            continue;
530                        }
531                        let matches = match catch_unwind(AssertUnwindSafe(|| matcher(&message))) {
532                            Ok(matches) => matches,
533                            Err(_) => return ResponseWait::Failed(ErrorKind::CallbackPanicked),
534                        };
535                        if matches {
536                            return ResponseWait::Matched;
537                        }
538                    }
539                    Ok(None) => return ResponseWait::Closed,
540                    // DropOldest is the safe default for a matcher queue. A
541                    // lag means unrelated messages were discarded; keep
542                    // waiting for the current connection's matching response
543                    // instead of turning normal market-data pressure into a
544                    // socket failure.
545                    Err(ReceiveError::Lagged { .. }) => continue,
546                    Err(ReceiveError::Failed(error)) => {
547                        return ResponseWait::Failed(error.kind());
548                    }
549                }
550            }
551        }
552    }
553}
554
555fn send_options(timeout: Duration) -> Result<SendOptions> {
556    let deadline = StdInstant::now().checked_add(timeout).ok_or_else(|| {
557        NetError::config(
558            "application_heartbeat.send_timeout",
559            "cannot be represented as a deadline",
560        )
561    })?;
562    Ok(SendOptions {
563        disconnected: DisconnectedPolicy::WaitForReconnect,
564        enqueue_timeout: Some(timeout),
565        write_timeout: timeout,
566        deadline: Some(deadline),
567        ..SendOptions::default()
568    })
569}
570
571fn record_failure(
572    snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
573    error: &NetError,
574    action: ApplicationHeartbeatFailureAction,
575) -> bool {
576    record_failure_kind(snapshot, error.kind(), action)
577}
578
579fn record_failure_kind(
580    snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
581    kind: ErrorKind,
582    action: ApplicationHeartbeatFailureAction,
583) -> bool {
584    snapshot_update(snapshot, |current| {
585        increment(&mut current.failures);
586        current.last_error = Some(kind);
587        current.state = ApplicationHeartbeatState::Running;
588    });
589    action == ApplicationHeartbeatFailureAction::Stop
590}
591
592fn increment(value: &mut u64) {
593    if let Some(next) = value.checked_add(1) {
594        *value = next;
595    }
596}
597
598fn snapshot_copy(
599    snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
600) -> ApplicationHeartbeatSnapshot {
601    *lock_snapshot(snapshot)
602}
603
604fn snapshot_update<F>(snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>, update: F)
605where
606    F: FnOnce(&mut ApplicationHeartbeatSnapshot),
607{
608    let mut current = lock_snapshot(snapshot);
609    update(&mut current);
610}
611
612fn lock_snapshot<'a>(
613    snapshot: &'a Arc<Mutex<ApplicationHeartbeatSnapshot>>,
614) -> MutexGuard<'a, ApplicationHeartbeatSnapshot> {
615    match snapshot.lock() {
616        Ok(guard) => guard,
617        Err(poisoned) => {
618            crate::log_e!(crate::common::log::log_def::LogType::WSC;
619                "application_heartbeat_snapshot", "error", "lock_poisoned_recovered");
620            poisoned.into_inner()
621        }
622    }
623}
624
625#[cfg(test)]
626#[path = "application_heartbeat_tests.rs"]
627mod tests;