Skip to main content

open_net/api/ws/
application_heartbeat.rs

1//! Application-level WebSocket heartbeat support.
2//!
3//! WebSocket control Ping/Pong frames are handled by the transport heartbeat.
4//! This module provides a separate, bounded loop for exchanges that require a
5//! text or binary heartbeat message at the application protocol level.
6
7use super::{
8    DisconnectedPolicy, IncomingMessage, Message, MessageReceiver, ReceiveOptions, ReceiveOverflow,
9    SendOptions, Sender,
10};
11use crate::error::{ErrorKind, ErrorStage, ReceiveError};
12use crate::module::ws_client::session_runtime::SessionRuntime;
13use crate::{NetError, Result};
14use std::panic::{catch_unwind, AssertUnwindSafe};
15use std::sync::{Arc, Mutex, MutexGuard};
16use std::time::{Duration, Instant as StdInstant, SystemTime};
17use tokio::task::JoinHandle;
18use tokio::time::{self, Instant};
19use tokio_util::sync::CancellationToken;
20
21/// Callback used to recognize an application-level heartbeat response.
22///
23/// The callback must only inspect the immutable message. A callback panic is
24/// converted to [`ErrorKind::CallbackPanicked`] and never escapes the task.
25/// Matching is message-level and connection-scoped; protocols that can deliver
26/// delayed or repeated responses should include a request nonce in the payload
27/// and verify it in this callback.
28pub type ApplicationHeartbeatMatcher =
29    Arc<dyn Fn(&IncomingMessage) -> bool + Send + Sync + 'static>;
30
31/// Action to take after an application heartbeat send or response failure.
32#[derive(Clone, Copy, Debug, Eq, PartialEq)]
33pub enum ApplicationHeartbeatFailureAction {
34    /// Record the failure and continue with the next scheduled heartbeat.
35    Continue,
36    /// Record the failure and stop the application heartbeat loop.
37    Stop,
38}
39
40/// Configuration for a fixed-interval application heartbeat loop.
41#[derive(Clone)]
42pub struct ApplicationHeartbeatConfig {
43    /// Delay between heartbeat attempts. The first heartbeat is sent after this delay.
44    pub interval: Duration,
45    /// Text or binary payload sent on every heartbeat attempt.
46    pub payload: Message,
47    /// Maximum time allowed for queue admission and the local write.
48    pub send_timeout: Duration,
49    /// How long to wait for a matching response after a successful write.
50    /// `None` disables response matching and timeout accounting.
51    pub response_timeout: Option<Duration>,
52    /// Number of consecutive missed responses tolerated before `Stop` takes effect.
53    pub max_missed_responses: u32,
54    /// Optional application response matcher. When absent, writes are observed only.
55    pub matcher: Option<ApplicationHeartbeatMatcher>,
56    /// Receive queue used exclusively by the heartbeat loop when a matcher is set.
57    /// The default is [`ReceiveOverflow::DropOldest`] so unrelated business
58    /// traffic cannot close the session; selecting `Disconnect` makes queue
59    /// overflow a heartbeat receiver failure by design.
60    pub receive_options: ReceiveOptions,
61    /// Failure handling policy for writes, receiver failures, and response timeouts.
62    pub failure_action: ApplicationHeartbeatFailureAction,
63}
64
65impl ApplicationHeartbeatConfig {
66    /// Creates a fixed-interval heartbeat configuration with no response matching.
67    pub fn new(interval: Duration, payload: impl Into<Message>) -> Self {
68        Self {
69            interval,
70            payload: payload.into(),
71            send_timeout: Duration::from_secs(10),
72            response_timeout: None,
73            max_missed_responses: 1,
74            matcher: None,
75            receive_options: ReceiveOptions {
76                overflow: ReceiveOverflow::DropOldest,
77                ..ReceiveOptions::default()
78            },
79            failure_action: ApplicationHeartbeatFailureAction::Continue,
80        }
81    }
82
83    /// Sets the timeout used for queue admission and local WebSocket writing.
84    pub fn with_send_timeout(mut self, timeout: Duration) -> Self {
85        self.send_timeout = timeout;
86        self
87    }
88
89    /// Enables response matching with the supplied timeout and matcher.
90    pub fn with_response_matcher(
91        mut self,
92        timeout: Duration,
93        matcher: impl Fn(&IncomingMessage) -> bool + Send + Sync + 'static,
94    ) -> Self {
95        self.response_timeout = Some(timeout);
96        self.matcher = Some(Arc::new(matcher));
97        self
98    }
99
100    /// Sets the maximum number of consecutive response timeouts.
101    pub fn with_max_missed_responses(mut self, max_missed_responses: u32) -> Self {
102        self.max_missed_responses = max_missed_responses;
103        self
104    }
105
106    /// Sets the receive queue options used by the heartbeat loop.
107    pub fn with_receive_options(mut self, receive_options: ReceiveOptions) -> Self {
108        self.receive_options = receive_options;
109        self
110    }
111
112    /// Sets whether failures stop or leave the loop running.
113    pub fn with_failure_action(
114        mut self,
115        failure_action: ApplicationHeartbeatFailureAction,
116    ) -> Self {
117        self.failure_action = failure_action;
118        self
119    }
120
121    /// Validates values that would otherwise create a busy loop or an unbounded wait.
122    pub fn validate(&self) -> Result<()> {
123        if self.interval.is_zero() {
124            return Err(NetError::config(
125                "application_heartbeat.interval",
126                "must be greater than zero",
127            ));
128        }
129        if Instant::now().checked_add(self.interval).is_none() {
130            return Err(NetError::config(
131                "application_heartbeat.interval",
132                "cannot be represented as a timer deadline",
133            ));
134        }
135        if self.send_timeout.is_zero() {
136            return Err(NetError::config(
137                "application_heartbeat.send_timeout",
138                "must be greater than zero",
139            ));
140        }
141        if StdInstant::now().checked_add(self.send_timeout).is_none() {
142            return Err(NetError::config(
143                "application_heartbeat.send_timeout",
144                "cannot be represented as a deadline",
145            ));
146        }
147        if self.max_missed_responses == 0 {
148            return Err(NetError::config(
149                "application_heartbeat.max_missed_responses",
150                "must be greater than zero",
151            ));
152        }
153        if self.matcher.is_some() && self.receive_options.include_control_frames {
154            return Err(NetError::config(
155                "application_heartbeat.receive_options.include_control_frames",
156                "must be false so control Pong frames cannot acknowledge application heartbeats",
157            ));
158        }
159        if self.matcher.is_some() && self.response_timeout.is_none() {
160            return Err(NetError::config(
161                "application_heartbeat.response_timeout",
162                "is required when matcher is set",
163            ));
164        }
165        if let Some(timeout) = self.response_timeout {
166            if StdInstant::now().checked_add(timeout).is_none() {
167                return Err(NetError::config(
168                    "application_heartbeat.response_timeout",
169                    "must fit an absolute deadline",
170                ));
171            }
172            if timeout.is_zero() {
173                return Err(NetError::config(
174                    "application_heartbeat.response_timeout",
175                    "must be greater than zero",
176                ));
177            }
178            if Instant::now().checked_add(timeout).is_none() {
179                return Err(NetError::config(
180                    "application_heartbeat.response_timeout",
181                    "cannot be represented as a deadline",
182                ));
183            }
184            if self.matcher.is_none() {
185                return Err(NetError::config(
186                    "application_heartbeat.matcher",
187                    "is required when response_timeout is set",
188                ));
189            }
190        }
191        if self.matcher.is_some() {
192            self.receive_options.validate()?;
193        }
194        Ok(())
195    }
196}
197
198impl std::fmt::Debug for ApplicationHeartbeatConfig {
199    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
200        formatter
201            .debug_struct("ApplicationHeartbeatConfig")
202            .field("interval", &self.interval)
203            .field("payload_bytes", &self.payload.len())
204            .field("send_timeout", &self.send_timeout)
205            .field("response_timeout", &self.response_timeout)
206            .field("max_missed_responses", &self.max_missed_responses)
207            .field("has_matcher", &self.matcher.is_some())
208            .field("receive_options", &self.receive_options)
209            .field("failure_action", &self.failure_action)
210            .finish()
211    }
212}
213
214/// Current state of an application heartbeat loop.
215#[derive(Clone, Copy, Debug, Eq, PartialEq)]
216pub enum ApplicationHeartbeatState {
217    /// The loop is waiting for its next scheduled send.
218    Running,
219    /// A heartbeat was written and a response is being matched.
220    WaitingForResponse,
221    /// The loop has stopped because its handle was stopped, its session ended, or policy stopped it.
222    Stopped,
223}
224
225/// Immutable counters and last-error classification for an application heartbeat.
226#[derive(Clone, Copy, Debug, Eq, PartialEq)]
227pub struct ApplicationHeartbeatSnapshot {
228    /// Current loop state.
229    pub state: ApplicationHeartbeatState,
230    /// Number of successful local writes.
231    pub sent: u64,
232    /// Number of matched application responses.
233    pub matched_responses: u64,
234    /// Number of response deadlines reached without a match.
235    pub missed_responses: u64,
236    /// Number of send, receive, matcher, or response failures.
237    pub failures: u64,
238    /// Classification of the most recent failure, if any.
239    pub last_error: Option<ErrorKind>,
240}
241
242impl Default for ApplicationHeartbeatSnapshot {
243    fn default() -> Self {
244        Self {
245            state: ApplicationHeartbeatState::Running,
246            sent: 0,
247            matched_responses: 0,
248            missed_responses: 0,
249            failures: 0,
250            last_error: None,
251        }
252    }
253}
254
255/// Controls one application heartbeat loop.
256///
257/// Dropping the handle requests cancellation and aborts only the heartbeat task;
258/// the owning WebSocket session remains alive.
259pub struct ApplicationHeartbeatHandle {
260    cancellation: CancellationToken,
261    snapshot: Arc<Mutex<ApplicationHeartbeatSnapshot>>,
262    task: Option<JoinHandle<()>>,
263}
264
265impl ApplicationHeartbeatHandle {
266    pub(crate) fn new(
267        sender: Sender,
268        receiver: Option<MessageReceiver>,
269        config: ApplicationHeartbeatConfig,
270    ) -> Result<Self> {
271        config.validate()?;
272        let cancellation = CancellationToken::new();
273        let snapshot = Arc::new(Mutex::new(ApplicationHeartbeatSnapshot::default()));
274        let runtime = sender.runtime.clone();
275        let task = runtime.clone().spawn(run_heartbeat(
276            sender,
277            receiver,
278            config,
279            cancellation.clone(),
280            snapshot.clone(),
281            runtime,
282        ))?;
283        Ok(Self {
284            cancellation,
285            snapshot,
286            task: Some(task),
287        })
288    }
289
290    /// Requests the loop to stop. The request is idempotent.
291    pub fn stop(&self) {
292        self.cancellation.cancel();
293    }
294
295    /// Returns a point-in-time snapshot without waiting for the loop.
296    pub fn snapshot(&self) -> ApplicationHeartbeatSnapshot {
297        snapshot_copy(&self.snapshot)
298    }
299
300    /// Waits for the loop task to finish and returns its final snapshot.
301    pub async fn stopped(mut self) -> Result<ApplicationHeartbeatSnapshot> {
302        let task = self
303            .task
304            .take()
305            .ok_or_else(|| NetError::from(ErrorKind::Internal).with_stage(ErrorStage::Runtime))?;
306        task.await.map_err(|error| {
307            NetError::with_source(ErrorKind::EngineDropped, error).with_stage(ErrorStage::Runtime)
308        })?;
309        Ok(snapshot_copy(&self.snapshot))
310    }
311}
312
313impl Drop for ApplicationHeartbeatHandle {
314    fn drop(&mut self) {
315        self.cancellation.cancel();
316        if let Some(task) = self.task.take() {
317            task.abort();
318        }
319    }
320}
321
322#[derive(Clone, Copy, Debug, Eq, PartialEq)]
323enum ResponseWait {
324    Matched,
325    TimedOut,
326    Closed,
327    Cancelled,
328    Failed(HeartbeatFailure),
329}
330
331#[derive(Clone, Copy, Debug, Eq, PartialEq)]
332enum HeartbeatPhase {
333    SendOptions,
334    Send,
335    Connection,
336    ResponseWait,
337    Matcher,
338    Receive,
339}
340
341impl HeartbeatPhase {
342    fn name(self) -> &'static str {
343        match self {
344            Self::SendOptions => "send_options",
345            Self::Send => "send",
346            Self::Connection => "connection",
347            Self::ResponseWait => "response_wait",
348            Self::Matcher => "matcher",
349            Self::Receive => "receive",
350        }
351    }
352}
353
354#[derive(Clone, Copy, Debug, Eq, PartialEq)]
355struct HeartbeatFailure {
356    phase: HeartbeatPhase,
357    kind: ErrorKind,
358    stage: Option<ErrorStage>,
359}
360
361const HEARTBEAT_STAGE_SLOTS: usize = ErrorStage::Runtime as usize + 2;
362
363#[derive(Default)]
364struct HeartbeatDiagnostics {
365    client_id: u64,
366    last: Option<HeartbeatFailure>,
367    failures: u64,
368    // One bit per SDK ErrorKind for each fixed phase/stage, including no stage.
369    // None of these slots are keyed by user-controlled data.
370    reported: [[u64; HEARTBEAT_STAGE_SLOTS]; 6],
371}
372
373impl HeartbeatDiagnostics {
374    fn should_report(&mut self, failure: HeartbeatFailure, enabled: bool) -> bool {
375        if matches!(failure.kind, ErrorKind::Cancelled | ErrorKind::Closed) {
376            self.last = None;
377            self.failures = 0;
378            self.reported = [[0; HEARTBEAT_STAGE_SLOTS]; 6];
379            return false;
380        }
381        self.last = Some(failure);
382        self.failures = self.failures.saturating_add(1);
383        if !enabled {
384            // An unobserved failure does not spend a later subscriber's first
385            // diagnostic. Queue rejection retains the logger's drop semantics.
386            self.reported = [[0; HEARTBEAT_STAGE_SLOTS]; 6];
387            return false;
388        }
389        // ErrorKind currently has 40 data-free variants. Keep future out-of-range
390        // classifications bounded as well, without risking a diagnostic panic.
391        let bit = 1_u64
392            .checked_shl(failure.kind as u32)
393            .unwrap_or(1_u64 << 63);
394        let stage = failure
395            .stage
396            .map_or(0, |stage| stage as usize + 1)
397            .min(HEARTBEAT_STAGE_SLOTS - 1);
398        let reported = &mut self.reported[failure.phase as usize][stage];
399        let first = *reported & bit == 0;
400        *reported |= bit;
401        first
402    }
403
404    fn failed(&mut self, session_id: u64, failure: HeartbeatFailure) {
405        if self.should_report(
406            failure,
407            crate::OpenNetLogger::is_enabled_for(crate::LogType::WSC, crate::LogLevel::Error),
408        ) {
409            crate::log_e!(crate::LogType::WSC; "ws.heartbeat.failed",
410                "client_id|session_id|phase|kind|stage|failures", self.client_id, session_id, failure.phase.name(),
411                format!("{:?}", failure.kind), failure.stage.map(|stage| format!("{stage:?}")), self.failures);
412        }
413    }
414
415    fn error(&mut self, session_id: u64, phase: HeartbeatPhase, error: &NetError) {
416        self.failed(
417            session_id,
418            HeartbeatFailure {
419                phase,
420                kind: error.kind(),
421                stage: error.context().stage,
422            },
423        );
424    }
425
426    fn recovered(&mut self, session_id: u64) {
427        if let Some(failure) = self.last.take() {
428            crate::log_s!(crate::LogType::WSC, crate::LogLevel::Key; "ws.heartbeat.recovered",
429                "client_id|session_id|phase|failures", self.client_id, session_id, failure.phase.name(), self.failures);
430        }
431        self.failures = 0;
432        self.reported = [[0; HEARTBEAT_STAGE_SLOTS]; 6];
433    }
434
435    fn stopped(&self, session_id: u64) {
436        if let Some(failure) = self.last {
437            crate::log_s!(crate::LogType::WSC, crate::LogLevel::Key; "ws.heartbeat.stopped",
438                "client_id|session_id|phase|kind|failures", self.client_id, session_id, failure.phase.name(), format!("{:?}", failure.kind), self.failures);
439        }
440    }
441}
442
443async fn run_heartbeat(
444    sender: Sender,
445    mut receiver: Option<MessageReceiver>,
446    config: ApplicationHeartbeatConfig,
447    cancellation: CancellationToken,
448    snapshot: Arc<Mutex<ApplicationHeartbeatSnapshot>>,
449    runtime: Arc<SessionRuntime>,
450) {
451    let now = Instant::now();
452    let first_tick = match now.checked_add(config.interval) {
453        Some(deadline) => deadline,
454        None => now,
455    };
456    let mut ticker = time::interval_at(first_tick, config.interval);
457    ticker.set_missed_tick_behavior(time::MissedTickBehavior::Skip);
458    let mut consecutive_missed = 0_u32;
459    let mut stopped_by_policy = false;
460    let session_closed = runtime.closed_token();
461    let session_id = runtime.id.as_u64();
462    let mut diagnostics = HeartbeatDiagnostics {
463        client_id: runtime.client_id.as_u64(),
464        ..Default::default()
465    };
466
467    'heartbeat: loop {
468        tokio::select! {
469            _ = cancellation.cancelled() => break 'heartbeat,
470            _ = session_closed.cancelled() => break 'heartbeat,
471            _ = ticker.tick() => {
472                snapshot_update(&snapshot, |current| {
473                    current.state = ApplicationHeartbeatState::Running;
474                });
475                let options = match send_options(config.send_timeout) {
476                    Ok(options) => options,
477                    Err(error) => {
478                        diagnostics.error(session_id, HeartbeatPhase::SendOptions, &error);
479                        // A configuration/deadline failure is not a missed
480                        // response. Keep the response-timeout streak scoped to
481                        // actual response deadlines.
482                        consecutive_missed = 0;
483                        let should_stop = record_failure(
484                            &snapshot,
485                            &error,
486                            config.failure_action,
487                        );
488                        if should_stop {
489                            stopped_by_policy = true;
490                            break 'heartbeat;
491                        }
492                        continue;
493                    }
494                };
495                let sent_at = SystemTime::now();
496                let send = sender
497                    .message(config.payload.clone())
498                    .options(options)
499                    .send();
500                let send_result = tokio::select! {
501                    _ = cancellation.cancelled() => break 'heartbeat,
502                    _ = session_closed.cancelled() => break 'heartbeat,
503                    result = send => result,
504                };
505                if let Err(error) = send_result {
506                    diagnostics.error(session_id, HeartbeatPhase::Send, &error);
507                    // A write failure breaks the response-timeout streak; the
508                    // next successful write starts a new response sequence.
509                    consecutive_missed = 0;
510                    let should_stop = record_failure(
511                        &snapshot,
512                        &error,
513                        config.failure_action,
514                    );
515                    if should_stop {
516                        stopped_by_policy = true;
517                        break 'heartbeat;
518                    }
519                    continue;
520                }
521                snapshot_update(&snapshot, |current| {
522                    increment(&mut current.sent);
523                });
524                let (Some(matcher), Some(receiver)) =
525                    (config.matcher.as_ref(), receiver.as_mut())
526                else {
527                    diagnostics.recovered(session_id);
528                    continue;
529                };
530                let connection_id = match runtime.current_connection_id() {
531                    Ok(Some(connection_id)) => connection_id,
532                    Ok(None) => {
533                        // The write completed while the physical connection
534                        // was being retired or replaced. Do not turn this
535                        // lifecycle race into a response timeout for the new
536                        // connection; retry on the next interval instead.
537                        consecutive_missed = 0;
538                        let error = NetError::from(ErrorKind::NotConnected)
539                            .with_stage(ErrorStage::Runtime);
540                        let should_stop = record_failure(
541                            &snapshot,
542                            &error,
543                            config.failure_action,
544                        );
545                        if should_stop {
546                            // The retirement race is expected, but a policy stop
547                            // still names its actual cause instead of an older fault.
548                            diagnostics.last = Some(HeartbeatFailure {
549                                phase: HeartbeatPhase::Connection,
550                                kind: ErrorKind::NotConnected,
551                                stage: Some(ErrorStage::Runtime),
552                            });
553                            diagnostics.failures = diagnostics.failures.saturating_add(1);
554                            stopped_by_policy = true;
555                            break 'heartbeat;
556                        }
557                        continue;
558                    }
559                    Err(error) => {
560                        diagnostics.error(session_id, HeartbeatPhase::Connection, &error);
561                        // A connection/lifecycle lookup failure is not a
562                        // response timeout and must not carry a stale streak
563                        // into the next heartbeat.
564                        consecutive_missed = 0;
565                        let should_stop = record_failure(
566                            &snapshot,
567                            &error,
568                            config.failure_action,
569                        );
570                        if should_stop {
571                            stopped_by_policy = true;
572                            break 'heartbeat;
573                        }
574                        continue;
575                    }
576                };
577                snapshot_update(&snapshot, |current| {
578                    current.state = ApplicationHeartbeatState::WaitingForResponse;
579                });
580                let timeout = match config.response_timeout {
581                    Some(timeout) => timeout,
582                    None => continue,
583                };
584                match wait_for_response(
585                    receiver,
586                    matcher,
587                    timeout,
588                    &cancellation,
589                    &session_closed,
590                    Some(connection_id),
591                    sent_at,
592                ).await {
593                    ResponseWait::Matched => {
594                        consecutive_missed = 0;
595                        snapshot_update(&snapshot, |current| {
596                            increment(&mut current.matched_responses);
597                            current.state = ApplicationHeartbeatState::Running;
598                        });
599                        diagnostics.recovered(session_id);
600                    }
601                    ResponseWait::TimedOut => {
602                        diagnostics.failed(session_id, HeartbeatFailure {
603                            phase: HeartbeatPhase::ResponseWait, kind: ErrorKind::TimedOut, stage: Some(ErrorStage::Receive),
604                        });
605                        consecutive_missed = consecutive_missed.saturating_add(1);
606                        snapshot_update(&snapshot, |current| {
607                            increment(&mut current.missed_responses);
608                            current.last_error = Some(ErrorKind::TimedOut);
609                            increment(&mut current.failures);
610                            current.state = ApplicationHeartbeatState::Running;
611                        });
612                        if config.failure_action == ApplicationHeartbeatFailureAction::Stop
613                            && consecutive_missed >= config.max_missed_responses
614                        {
615                            stopped_by_policy = true;
616                            break 'heartbeat;
617                        }
618                    }
619                    ResponseWait::Closed | ResponseWait::Cancelled => break 'heartbeat,
620                    ResponseWait::Failed(failure) => {
621                        diagnostics.failed(session_id, failure);
622                        // Receiver and matcher failures are distinct from a
623                        // response deadline, so they reset consecutive misses.
624                        consecutive_missed = 0;
625                        let should_stop = record_failure_kind(
626                            &snapshot,
627                            failure.kind,
628                            config.failure_action,
629                        );
630                        if should_stop {
631                            stopped_by_policy = true;
632                            break 'heartbeat;
633                        }
634                    }
635                }
636            }
637        }
638    }
639    if let Some(receiver) = receiver.as_ref() {
640        receiver.unsubscribe();
641    }
642    snapshot_update(&snapshot, |current| {
643        current.state = ApplicationHeartbeatState::Stopped;
644    });
645    if stopped_by_policy {
646        diagnostics.stopped(session_id);
647    }
648}
649
650async fn wait_for_response(
651    receiver: &mut MessageReceiver,
652    matcher: &ApplicationHeartbeatMatcher,
653    timeout: Duration,
654    cancellation: &CancellationToken,
655    session_closed: &CancellationToken,
656    connection_id: Option<crate::ws::ConnectionId>,
657    sent_at: SystemTime,
658) -> ResponseWait {
659    let deadline = time::sleep(timeout);
660    tokio::pin!(deadline);
661    loop {
662        tokio::select! {
663            _ = cancellation.cancelled() => return ResponseWait::Cancelled,
664            _ = session_closed.cancelled() => return ResponseWait::Closed,
665            _ = &mut deadline => return ResponseWait::TimedOut,
666            result = receiver.recv() => {
667                match result {
668                    Ok(Some(message)) => {
669                        if connection_id != Some(message.connection_id()) {
670                            continue;
671                        }
672                        if message.received_at().duration_since(sent_at).is_err() {
673                            continue;
674                        }
675                        let matches = match catch_unwind(AssertUnwindSafe(|| matcher(&message))) {
676                            Ok(matches) => matches,
677                            Err(_) => return ResponseWait::Failed(HeartbeatFailure {
678                                phase: HeartbeatPhase::Matcher, kind: ErrorKind::CallbackPanicked, stage: Some(ErrorStage::Receive),
679                            }),
680                        };
681                        if matches {
682                            return ResponseWait::Matched;
683                        }
684                    }
685                    Ok(None) => return ResponseWait::Closed,
686                    // DropOldest is the safe default for a matcher queue. A
687                    // lag means unrelated messages were discarded; keep
688                    // waiting for the current connection's matching response
689                    // instead of turning normal market-data pressure into a
690                    // socket failure.
691                    Err(ReceiveError::Lagged { .. }) => continue,
692                    Err(ReceiveError::Failed(error)) => {
693                        return ResponseWait::Failed(HeartbeatFailure {
694                            phase: HeartbeatPhase::Receive, kind: error.kind(), stage: error.context().stage,
695                        });
696                    }
697                }
698            }
699        }
700    }
701}
702
703fn send_options(timeout: Duration) -> Result<SendOptions> {
704    let deadline = StdInstant::now().checked_add(timeout).ok_or_else(|| {
705        NetError::config(
706            "application_heartbeat.send_timeout",
707            "cannot be represented as a deadline",
708        )
709    })?;
710    Ok(SendOptions {
711        disconnected: DisconnectedPolicy::WaitForReconnect,
712        enqueue_timeout: Some(timeout),
713        write_timeout: timeout,
714        deadline: Some(deadline),
715        ..SendOptions::default()
716    })
717}
718
719fn record_failure(
720    snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
721    error: &NetError,
722    action: ApplicationHeartbeatFailureAction,
723) -> bool {
724    record_failure_kind(snapshot, error.kind(), action)
725}
726
727fn record_failure_kind(
728    snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
729    kind: ErrorKind,
730    action: ApplicationHeartbeatFailureAction,
731) -> bool {
732    snapshot_update(snapshot, |current| {
733        increment(&mut current.failures);
734        current.last_error = Some(kind);
735        current.state = ApplicationHeartbeatState::Running;
736    });
737    action == ApplicationHeartbeatFailureAction::Stop
738}
739
740fn increment(value: &mut u64) {
741    if let Some(next) = value.checked_add(1) {
742        *value = next;
743    }
744}
745
746fn snapshot_copy(
747    snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
748) -> ApplicationHeartbeatSnapshot {
749    *lock_snapshot(snapshot)
750}
751
752fn snapshot_update<F>(snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>, update: F)
753where
754    F: FnOnce(&mut ApplicationHeartbeatSnapshot),
755{
756    let mut current = lock_snapshot(snapshot);
757    update(&mut current);
758}
759
760fn lock_snapshot<'a>(
761    snapshot: &'a Arc<Mutex<ApplicationHeartbeatSnapshot>>,
762) -> MutexGuard<'a, ApplicationHeartbeatSnapshot> {
763    match snapshot.lock() {
764        Ok(guard) => guard,
765        Err(poisoned) => {
766            crate::log_e!(crate::common::log::log_def::LogType::WSC;
767                "ws.heartbeat.lock_snapshot", "arg_snapshot_present|error", true, "lock_poisoned_recovered");
768            poisoned.into_inner()
769        }
770    }
771}
772
773#[cfg(test)]
774#[path = "application_heartbeat_tests.rs"]
775mod tests;