Skip to main content

open_net/api/ws/
application_heartbeat.rs

1//! Application-level WebSocket heartbeat support.
2//!
3//! WebSocket control Ping/Pong frames are handled by the transport heartbeat.
4//! This module provides a separate, bounded loop for exchanges that require a
5//! text or binary heartbeat message at the application protocol level.
6//!
7//! These are ordinary data messages: they share application queue admission and
8//! wait for an already-started fragmented message to finish. They do not have
9//! the interleaving privileges of RFC 6455 Ping/Pong control frames. If the
10//! application protocol needs scheduling opportunities within a large payload,
11//! it must define multiple complete messages or use a separate connection; SDK
12//! frame fragmentation preserves the original logical message.
13
14use super::{
15    DisconnectedPolicy, IncomingMessage, Message, MessageReceiver, ReceiveOptions, ReceiveOverflow,
16    SendOptions, Sender,
17};
18use crate::error::{ErrorKind, ErrorStage, ReceiveError};
19use crate::module::ws_client::session_runtime::SessionRuntime;
20use crate::{NetError, Result};
21use std::panic::{catch_unwind, AssertUnwindSafe};
22use std::sync::{Arc, Mutex, MutexGuard};
23use std::time::{Duration, Instant as StdInstant, SystemTime};
24use tokio::task::JoinHandle;
25use tokio::time::{self, Instant};
26use tokio_util::sync::CancellationToken;
27
28/// Callback used to recognize an application-level heartbeat response.
29///
30/// The callback must only inspect the immutable message. A callback panic is
31/// converted to [`ErrorKind::CallbackPanicked`] and never escapes the task.
32/// Matching is message-level and connection-scoped; protocols that can deliver
33/// delayed or repeated responses should include a request nonce in the payload
34/// and verify it in this callback.
35pub type ApplicationHeartbeatMatcher =
36    Arc<dyn Fn(&IncomingMessage) -> bool + Send + Sync + 'static>;
37
38/// Action to take after an application heartbeat send or response failure.
39#[derive(Clone, Copy, Debug, Eq, PartialEq)]
40pub enum ApplicationHeartbeatFailureAction {
41    /// Record the failure and continue with the next scheduled heartbeat.
42    Continue,
43    /// Record the failure and stop the application heartbeat loop.
44    Stop,
45}
46
47/// Configuration for a fixed-interval application heartbeat loop.
48#[derive(Clone)]
49pub struct ApplicationHeartbeatConfig {
50    /// Delay between heartbeat attempts. The first heartbeat is sent after this delay.
51    pub interval: Duration,
52    /// Text or binary payload sent on every heartbeat attempt.
53    pub payload: Message,
54    /// Maximum time allowed for queue admission and the local write.
55    pub send_timeout: Duration,
56    /// How long to wait for a matching response after a successful write.
57    /// `None` disables response matching and timeout accounting.
58    pub response_timeout: Option<Duration>,
59    /// Number of consecutive missed responses tolerated before `Stop` takes effect.
60    pub max_missed_responses: u32,
61    /// Optional application response matcher. When absent, writes are observed only.
62    pub matcher: Option<ApplicationHeartbeatMatcher>,
63    /// Receive queue used exclusively by the heartbeat loop when a matcher is set.
64    /// The default is [`ReceiveOverflow::DropOldest`] so unrelated business
65    /// traffic cannot close the session; selecting `Disconnect` makes queue
66    /// overflow a heartbeat receiver failure by design.
67    pub receive_options: ReceiveOptions,
68    /// Failure handling policy for writes, receiver failures, and response timeouts.
69    pub failure_action: ApplicationHeartbeatFailureAction,
70}
71
72impl ApplicationHeartbeatConfig {
73    /// Creates a fixed-interval heartbeat configuration with no response matching.
74    pub fn new(interval: Duration, payload: impl Into<Message>) -> Self {
75        Self {
76            interval,
77            payload: payload.into(),
78            send_timeout: Duration::from_secs(10),
79            response_timeout: None,
80            max_missed_responses: 1,
81            matcher: None,
82            receive_options: ReceiveOptions {
83                overflow: ReceiveOverflow::DropOldest,
84                ..ReceiveOptions::default()
85            },
86            failure_action: ApplicationHeartbeatFailureAction::Continue,
87        }
88    }
89
90    /// Sets the timeout used for queue admission and local WebSocket writing.
91    pub fn with_send_timeout(mut self, timeout: Duration) -> Self {
92        self.send_timeout = timeout;
93        self
94    }
95
96    /// Enables response matching with the supplied timeout and matcher.
97    pub fn with_response_matcher(
98        mut self,
99        timeout: Duration,
100        matcher: impl Fn(&IncomingMessage) -> bool + Send + Sync + 'static,
101    ) -> Self {
102        self.response_timeout = Some(timeout);
103        self.matcher = Some(Arc::new(matcher));
104        self
105    }
106
107    /// Sets the maximum number of consecutive response timeouts.
108    pub fn with_max_missed_responses(mut self, max_missed_responses: u32) -> Self {
109        self.max_missed_responses = max_missed_responses;
110        self
111    }
112
113    /// Sets the receive queue options used by the heartbeat loop.
114    pub fn with_receive_options(mut self, receive_options: ReceiveOptions) -> Self {
115        self.receive_options = receive_options;
116        self
117    }
118
119    /// Sets whether failures stop or leave the loop running.
120    pub fn with_failure_action(
121        mut self,
122        failure_action: ApplicationHeartbeatFailureAction,
123    ) -> Self {
124        self.failure_action = failure_action;
125        self
126    }
127
128    /// Validates values that would otherwise create a busy loop or an unbounded wait.
129    pub fn validate(&self) -> Result<()> {
130        if self.interval.is_zero() {
131            return Err(NetError::config(
132                "application_heartbeat.interval",
133                "must be greater than zero",
134            ));
135        }
136        if Instant::now().checked_add(self.interval).is_none() {
137            return Err(NetError::config(
138                "application_heartbeat.interval",
139                "cannot be represented as a timer deadline",
140            ));
141        }
142        if self.send_timeout.is_zero() {
143            return Err(NetError::config(
144                "application_heartbeat.send_timeout",
145                "must be greater than zero",
146            ));
147        }
148        if StdInstant::now().checked_add(self.send_timeout).is_none() {
149            return Err(NetError::config(
150                "application_heartbeat.send_timeout",
151                "cannot be represented as a deadline",
152            ));
153        }
154        if self.max_missed_responses == 0 {
155            return Err(NetError::config(
156                "application_heartbeat.max_missed_responses",
157                "must be greater than zero",
158            ));
159        }
160        if self.matcher.is_some() && self.receive_options.include_control_frames {
161            return Err(NetError::config(
162                "application_heartbeat.receive_options.include_control_frames",
163                "must be false so control Pong frames cannot acknowledge application heartbeats",
164            ));
165        }
166        if self.matcher.is_some() && self.response_timeout.is_none() {
167            return Err(NetError::config(
168                "application_heartbeat.response_timeout",
169                "is required when matcher is set",
170            ));
171        }
172        if let Some(timeout) = self.response_timeout {
173            if StdInstant::now().checked_add(timeout).is_none() {
174                return Err(NetError::config(
175                    "application_heartbeat.response_timeout",
176                    "must fit an absolute deadline",
177                ));
178            }
179            if timeout.is_zero() {
180                return Err(NetError::config(
181                    "application_heartbeat.response_timeout",
182                    "must be greater than zero",
183                ));
184            }
185            if Instant::now().checked_add(timeout).is_none() {
186                return Err(NetError::config(
187                    "application_heartbeat.response_timeout",
188                    "cannot be represented as a deadline",
189                ));
190            }
191            if self.matcher.is_none() {
192                return Err(NetError::config(
193                    "application_heartbeat.matcher",
194                    "is required when response_timeout is set",
195                ));
196            }
197        }
198        if self.matcher.is_some() {
199            self.receive_options.validate()?;
200        }
201        Ok(())
202    }
203}
204
205impl std::fmt::Debug for ApplicationHeartbeatConfig {
206    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
207        formatter
208            .debug_struct("ApplicationHeartbeatConfig")
209            .field("interval", &self.interval)
210            .field("payload_bytes", &self.payload.len())
211            .field("send_timeout", &self.send_timeout)
212            .field("response_timeout", &self.response_timeout)
213            .field("max_missed_responses", &self.max_missed_responses)
214            .field("has_matcher", &self.matcher.is_some())
215            .field("receive_options", &self.receive_options)
216            .field("failure_action", &self.failure_action)
217            .finish()
218    }
219}
220
221/// Current state of an application heartbeat loop.
222#[derive(Clone, Copy, Debug, Eq, PartialEq)]
223pub enum ApplicationHeartbeatState {
224    /// The loop is waiting for its next scheduled send.
225    Running,
226    /// A heartbeat was written and a response is being matched.
227    WaitingForResponse,
228    /// The loop has stopped because its handle was stopped, its session ended, or policy stopped it.
229    Stopped,
230}
231
232/// Immutable counters and last-error classification for an application heartbeat.
233#[derive(Clone, Copy, Debug, Eq, PartialEq)]
234pub struct ApplicationHeartbeatSnapshot {
235    /// Current loop state.
236    pub state: ApplicationHeartbeatState,
237    /// Number of successful local writes.
238    pub sent: u64,
239    /// Number of matched application responses.
240    pub matched_responses: u64,
241    /// Number of response deadlines reached without a match.
242    pub missed_responses: u64,
243    /// Number of send, receive, matcher, or response failures.
244    pub failures: u64,
245    /// Classification of the most recent failure, if any.
246    pub last_error: Option<ErrorKind>,
247}
248
249impl Default for ApplicationHeartbeatSnapshot {
250    fn default() -> Self {
251        Self {
252            state: ApplicationHeartbeatState::Running,
253            sent: 0,
254            matched_responses: 0,
255            missed_responses: 0,
256            failures: 0,
257            last_error: None,
258        }
259    }
260}
261
262/// Controls one application heartbeat loop.
263///
264/// Dropping the handle requests cancellation and aborts only the heartbeat task;
265/// the owning WebSocket session remains alive.
266pub struct ApplicationHeartbeatHandle {
267    cancellation: CancellationToken,
268    snapshot: Arc<Mutex<ApplicationHeartbeatSnapshot>>,
269    task: Option<JoinHandle<()>>,
270}
271
272impl ApplicationHeartbeatHandle {
273    pub(crate) fn new(
274        sender: Sender,
275        receiver: Option<MessageReceiver>,
276        config: ApplicationHeartbeatConfig,
277    ) -> Result<Self> {
278        config.validate()?;
279        let cancellation = CancellationToken::new();
280        let snapshot = Arc::new(Mutex::new(ApplicationHeartbeatSnapshot::default()));
281        let runtime = sender.runtime.clone();
282        let task = runtime.clone().spawn(run_heartbeat(
283            sender,
284            receiver,
285            config,
286            cancellation.clone(),
287            snapshot.clone(),
288            runtime,
289        ))?;
290        Ok(Self {
291            cancellation,
292            snapshot,
293            task: Some(task),
294        })
295    }
296
297    /// Requests the loop to stop. The request is idempotent.
298    pub fn stop(&self) {
299        self.cancellation.cancel();
300    }
301
302    /// Returns a point-in-time snapshot without waiting for the loop.
303    pub fn snapshot(&self) -> ApplicationHeartbeatSnapshot {
304        snapshot_copy(&self.snapshot)
305    }
306
307    /// Waits for the loop task to finish and returns its final snapshot.
308    pub async fn stopped(mut self) -> Result<ApplicationHeartbeatSnapshot> {
309        let task = self
310            .task
311            .take()
312            .ok_or_else(|| NetError::from(ErrorKind::Internal).with_stage(ErrorStage::Runtime))?;
313        task.await.map_err(|error| {
314            NetError::with_source(ErrorKind::EngineDropped, error).with_stage(ErrorStage::Runtime)
315        })?;
316        Ok(snapshot_copy(&self.snapshot))
317    }
318}
319
320impl Drop for ApplicationHeartbeatHandle {
321    fn drop(&mut self) {
322        self.cancellation.cancel();
323        if let Some(task) = self.task.take() {
324            task.abort();
325        }
326    }
327}
328
329#[derive(Clone, Copy, Debug, Eq, PartialEq)]
330enum ResponseWait {
331    Matched,
332    TimedOut,
333    Closed,
334    Cancelled,
335    Failed(HeartbeatFailure),
336}
337
338#[derive(Clone, Copy, Debug, Eq, PartialEq)]
339enum HeartbeatPhase {
340    SendOptions,
341    Send,
342    Connection,
343    ResponseWait,
344    Matcher,
345    Receive,
346}
347
348impl HeartbeatPhase {
349    fn name(self) -> &'static str {
350        match self {
351            Self::SendOptions => "send_options",
352            Self::Send => "send",
353            Self::Connection => "connection",
354            Self::ResponseWait => "response_wait",
355            Self::Matcher => "matcher",
356            Self::Receive => "receive",
357        }
358    }
359}
360
361#[derive(Clone, Copy, Debug, Eq, PartialEq)]
362struct HeartbeatFailure {
363    phase: HeartbeatPhase,
364    kind: ErrorKind,
365    stage: Option<ErrorStage>,
366}
367
368const HEARTBEAT_STAGE_SLOTS: usize = ErrorStage::Runtime as usize + 2;
369
370#[derive(Default)]
371struct HeartbeatDiagnostics {
372    client_id: u64,
373    last: Option<HeartbeatFailure>,
374    failures: u64,
375    // One bit per SDK ErrorKind for each fixed phase/stage, including no stage.
376    // None of these slots are keyed by user-controlled data.
377    reported: [[u64; HEARTBEAT_STAGE_SLOTS]; 6],
378}
379
380impl HeartbeatDiagnostics {
381    fn should_report(&mut self, failure: HeartbeatFailure, enabled: bool) -> bool {
382        if matches!(failure.kind, ErrorKind::Cancelled | ErrorKind::Closed) {
383            self.last = None;
384            self.failures = 0;
385            self.reported = [[0; HEARTBEAT_STAGE_SLOTS]; 6];
386            return false;
387        }
388        self.last = Some(failure);
389        self.failures = self.failures.saturating_add(1);
390        if !enabled {
391            // An unobserved failure does not spend a later subscriber's first
392            // diagnostic. Queue rejection retains the logger's drop semantics.
393            self.reported = [[0; HEARTBEAT_STAGE_SLOTS]; 6];
394            return false;
395        }
396        // ErrorKind currently has 40 data-free variants. Keep future out-of-range
397        // classifications bounded as well, without risking a diagnostic panic.
398        let bit = 1_u64
399            .checked_shl(failure.kind as u32)
400            .unwrap_or(1_u64 << 63);
401        let stage = failure
402            .stage
403            .map_or(0, |stage| stage as usize + 1)
404            .min(HEARTBEAT_STAGE_SLOTS - 1);
405        let reported = &mut self.reported[failure.phase as usize][stage];
406        let first = *reported & bit == 0;
407        *reported |= bit;
408        first
409    }
410
411    fn failed(&mut self, session_id: u64, failure: HeartbeatFailure) {
412        if self.should_report(
413            failure,
414            crate::OpenNetLogger::is_enabled_for(crate::LogType::WSC, crate::LogLevel::Error),
415        ) {
416            crate::log_e!(crate::LogType::WSC; "ws.heartbeat.failed",
417                "client_id|session_id|phase|kind|stage|failures", self.client_id, session_id, failure.phase.name(),
418                format!("{:?}", failure.kind), failure.stage.map(|stage| format!("{stage:?}")), self.failures);
419        }
420    }
421
422    fn error(&mut self, session_id: u64, phase: HeartbeatPhase, error: &NetError) {
423        self.failed(
424            session_id,
425            HeartbeatFailure {
426                phase,
427                kind: error.kind(),
428                stage: error.context().stage,
429            },
430        );
431    }
432
433    fn recovered(&mut self, session_id: u64) {
434        if let Some(failure) = self.last.take() {
435            crate::log_s!(crate::LogType::WSC, crate::LogLevel::Key; "ws.heartbeat.recovered",
436                "client_id|session_id|phase|failures", self.client_id, session_id, failure.phase.name(), self.failures);
437        }
438        self.failures = 0;
439        self.reported = [[0; HEARTBEAT_STAGE_SLOTS]; 6];
440    }
441
442    fn stopped(&self, session_id: u64) {
443        if let Some(failure) = self.last {
444            crate::log_s!(crate::LogType::WSC, crate::LogLevel::Key; "ws.heartbeat.stopped",
445                "client_id|session_id|phase|kind|failures", self.client_id, session_id, failure.phase.name(), format!("{:?}", failure.kind), self.failures);
446        }
447    }
448}
449
450async fn run_heartbeat(
451    sender: Sender,
452    mut receiver: Option<MessageReceiver>,
453    config: ApplicationHeartbeatConfig,
454    cancellation: CancellationToken,
455    snapshot: Arc<Mutex<ApplicationHeartbeatSnapshot>>,
456    runtime: Arc<SessionRuntime>,
457) {
458    let now = Instant::now();
459    let first_tick = match now.checked_add(config.interval) {
460        Some(deadline) => deadline,
461        None => now,
462    };
463    let mut ticker = time::interval_at(first_tick, config.interval);
464    ticker.set_missed_tick_behavior(time::MissedTickBehavior::Skip);
465    let mut consecutive_missed = 0_u32;
466    let mut stopped_by_policy = false;
467    let session_closed = runtime.closed_token();
468    let session_id = runtime.id.as_u64();
469    let mut diagnostics = HeartbeatDiagnostics {
470        client_id: runtime.client_id.as_u64(),
471        ..Default::default()
472    };
473
474    'heartbeat: loop {
475        tokio::select! {
476            _ = cancellation.cancelled() => break 'heartbeat,
477            _ = session_closed.cancelled() => break 'heartbeat,
478            _ = ticker.tick() => {
479                snapshot_update(&snapshot, |current| {
480                    current.state = ApplicationHeartbeatState::Running;
481                });
482                let options = match send_options(config.send_timeout) {
483                    Ok(options) => options,
484                    Err(error) => {
485                        diagnostics.error(session_id, HeartbeatPhase::SendOptions, &error);
486                        // A configuration/deadline failure is not a missed
487                        // response. Keep the response-timeout streak scoped to
488                        // actual response deadlines.
489                        consecutive_missed = 0;
490                        let should_stop = record_failure(
491                            &snapshot,
492                            &error,
493                            config.failure_action,
494                        );
495                        if should_stop {
496                            stopped_by_policy = true;
497                            break 'heartbeat;
498                        }
499                        continue;
500                    }
501                };
502                let sent_at = SystemTime::now();
503                let send = sender
504                    .message(config.payload.clone())
505                    .options(options)
506                    .send();
507                let send_result = tokio::select! {
508                    _ = cancellation.cancelled() => break 'heartbeat,
509                    _ = session_closed.cancelled() => break 'heartbeat,
510                    result = send => result,
511                };
512                if let Err(error) = send_result {
513                    diagnostics.error(session_id, HeartbeatPhase::Send, &error);
514                    // A write failure breaks the response-timeout streak; the
515                    // next successful write starts a new response sequence.
516                    consecutive_missed = 0;
517                    let should_stop = record_failure(
518                        &snapshot,
519                        &error,
520                        config.failure_action,
521                    );
522                    if should_stop {
523                        stopped_by_policy = true;
524                        break 'heartbeat;
525                    }
526                    continue;
527                }
528                snapshot_update(&snapshot, |current| {
529                    increment(&mut current.sent);
530                });
531                let (Some(matcher), Some(receiver)) =
532                    (config.matcher.as_ref(), receiver.as_mut())
533                else {
534                    diagnostics.recovered(session_id);
535                    continue;
536                };
537                let connection_id = match runtime.current_connection_id() {
538                    Ok(Some(connection_id)) => connection_id,
539                    Ok(None) => {
540                        // The write completed while the physical connection
541                        // was being retired or replaced. Do not turn this
542                        // lifecycle race into a response timeout for the new
543                        // connection; retry on the next interval instead.
544                        consecutive_missed = 0;
545                        let error = NetError::from(ErrorKind::NotConnected)
546                            .with_stage(ErrorStage::Runtime);
547                        let should_stop = record_failure(
548                            &snapshot,
549                            &error,
550                            config.failure_action,
551                        );
552                        if should_stop {
553                            // The retirement race is expected, but a policy stop
554                            // still names its actual cause instead of an older fault.
555                            diagnostics.last = Some(HeartbeatFailure {
556                                phase: HeartbeatPhase::Connection,
557                                kind: ErrorKind::NotConnected,
558                                stage: Some(ErrorStage::Runtime),
559                            });
560                            diagnostics.failures = diagnostics.failures.saturating_add(1);
561                            stopped_by_policy = true;
562                            break 'heartbeat;
563                        }
564                        continue;
565                    }
566                    Err(error) => {
567                        diagnostics.error(session_id, HeartbeatPhase::Connection, &error);
568                        // A connection/lifecycle lookup failure is not a
569                        // response timeout and must not carry a stale streak
570                        // into the next heartbeat.
571                        consecutive_missed = 0;
572                        let should_stop = record_failure(
573                            &snapshot,
574                            &error,
575                            config.failure_action,
576                        );
577                        if should_stop {
578                            stopped_by_policy = true;
579                            break 'heartbeat;
580                        }
581                        continue;
582                    }
583                };
584                snapshot_update(&snapshot, |current| {
585                    current.state = ApplicationHeartbeatState::WaitingForResponse;
586                });
587                let timeout = match config.response_timeout {
588                    Some(timeout) => timeout,
589                    None => continue,
590                };
591                match wait_for_response(
592                    receiver,
593                    matcher,
594                    timeout,
595                    &cancellation,
596                    &session_closed,
597                    Some(connection_id),
598                    sent_at,
599                ).await {
600                    ResponseWait::Matched => {
601                        consecutive_missed = 0;
602                        snapshot_update(&snapshot, |current| {
603                            increment(&mut current.matched_responses);
604                            current.state = ApplicationHeartbeatState::Running;
605                        });
606                        diagnostics.recovered(session_id);
607                    }
608                    ResponseWait::TimedOut => {
609                        diagnostics.failed(session_id, HeartbeatFailure {
610                            phase: HeartbeatPhase::ResponseWait, kind: ErrorKind::TimedOut, stage: Some(ErrorStage::Receive),
611                        });
612                        consecutive_missed = consecutive_missed.saturating_add(1);
613                        snapshot_update(&snapshot, |current| {
614                            increment(&mut current.missed_responses);
615                            current.last_error = Some(ErrorKind::TimedOut);
616                            increment(&mut current.failures);
617                            current.state = ApplicationHeartbeatState::Running;
618                        });
619                        if config.failure_action == ApplicationHeartbeatFailureAction::Stop
620                            && consecutive_missed >= config.max_missed_responses
621                        {
622                            stopped_by_policy = true;
623                            break 'heartbeat;
624                        }
625                    }
626                    ResponseWait::Closed | ResponseWait::Cancelled => break 'heartbeat,
627                    ResponseWait::Failed(failure) => {
628                        diagnostics.failed(session_id, failure);
629                        // Receiver and matcher failures are distinct from a
630                        // response deadline, so they reset consecutive misses.
631                        consecutive_missed = 0;
632                        let should_stop = record_failure_kind(
633                            &snapshot,
634                            failure.kind,
635                            config.failure_action,
636                        );
637                        if should_stop {
638                            stopped_by_policy = true;
639                            break 'heartbeat;
640                        }
641                    }
642                }
643            }
644        }
645    }
646    if let Some(receiver) = receiver.as_ref() {
647        receiver.unsubscribe();
648    }
649    snapshot_update(&snapshot, |current| {
650        current.state = ApplicationHeartbeatState::Stopped;
651    });
652    if stopped_by_policy {
653        diagnostics.stopped(session_id);
654    }
655}
656
657async fn wait_for_response(
658    receiver: &mut MessageReceiver,
659    matcher: &ApplicationHeartbeatMatcher,
660    timeout: Duration,
661    cancellation: &CancellationToken,
662    session_closed: &CancellationToken,
663    connection_id: Option<crate::ws::ConnectionId>,
664    sent_at: SystemTime,
665) -> ResponseWait {
666    let deadline = time::sleep(timeout);
667    tokio::pin!(deadline);
668    loop {
669        tokio::select! {
670            _ = cancellation.cancelled() => return ResponseWait::Cancelled,
671            _ = session_closed.cancelled() => return ResponseWait::Closed,
672            _ = &mut deadline => return ResponseWait::TimedOut,
673            result = receiver.recv() => {
674                match result {
675                    Ok(Some(message)) => {
676                        if connection_id != Some(message.connection_id()) {
677                            continue;
678                        }
679                        if message.received_at().duration_since(sent_at).is_err() {
680                            continue;
681                        }
682                        let matches = match catch_unwind(AssertUnwindSafe(|| matcher(&message))) {
683                            Ok(matches) => matches,
684                            Err(_) => return ResponseWait::Failed(HeartbeatFailure {
685                                phase: HeartbeatPhase::Matcher, kind: ErrorKind::CallbackPanicked, stage: Some(ErrorStage::Receive),
686                            }),
687                        };
688                        if matches {
689                            return ResponseWait::Matched;
690                        }
691                    }
692                    Ok(None) => return ResponseWait::Closed,
693                    // DropOldest is the safe default for a matcher queue. A
694                    // lag means unrelated messages were discarded; keep
695                    // waiting for the current connection's matching response
696                    // instead of turning normal market-data pressure into a
697                    // socket failure.
698                    Err(ReceiveError::Lagged { .. }) => continue,
699                    Err(ReceiveError::Failed(error)) => {
700                        return ResponseWait::Failed(HeartbeatFailure {
701                            phase: HeartbeatPhase::Receive, kind: error.kind(), stage: error.context().stage,
702                        });
703                    }
704                }
705            }
706        }
707    }
708}
709
710fn send_options(timeout: Duration) -> Result<SendOptions> {
711    let deadline = StdInstant::now().checked_add(timeout).ok_or_else(|| {
712        NetError::config(
713            "application_heartbeat.send_timeout",
714            "cannot be represented as a deadline",
715        )
716    })?;
717    Ok(SendOptions {
718        disconnected: DisconnectedPolicy::WaitForReconnect,
719        enqueue_timeout: Some(timeout),
720        write_timeout: timeout,
721        deadline: Some(deadline),
722        ..SendOptions::default()
723    })
724}
725
726fn record_failure(
727    snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
728    error: &NetError,
729    action: ApplicationHeartbeatFailureAction,
730) -> bool {
731    record_failure_kind(snapshot, error.kind(), action)
732}
733
734fn record_failure_kind(
735    snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
736    kind: ErrorKind,
737    action: ApplicationHeartbeatFailureAction,
738) -> bool {
739    snapshot_update(snapshot, |current| {
740        increment(&mut current.failures);
741        current.last_error = Some(kind);
742        current.state = ApplicationHeartbeatState::Running;
743    });
744    action == ApplicationHeartbeatFailureAction::Stop
745}
746
747fn increment(value: &mut u64) {
748    if let Some(next) = value.checked_add(1) {
749        *value = next;
750    }
751}
752
753fn snapshot_copy(
754    snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
755) -> ApplicationHeartbeatSnapshot {
756    *lock_snapshot(snapshot)
757}
758
759fn snapshot_update<F>(snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>, update: F)
760where
761    F: FnOnce(&mut ApplicationHeartbeatSnapshot),
762{
763    let mut current = lock_snapshot(snapshot);
764    update(&mut current);
765}
766
767fn lock_snapshot<'a>(
768    snapshot: &'a Arc<Mutex<ApplicationHeartbeatSnapshot>>,
769) -> MutexGuard<'a, ApplicationHeartbeatSnapshot> {
770    match snapshot.lock() {
771        Ok(guard) => guard,
772        Err(poisoned) => {
773            crate::log_e!(crate::common::log::log_def::LogType::WSC;
774                "ws.heartbeat.lock_snapshot", "arg_snapshot_present|error", true, "lock_poisoned_recovered");
775            poisoned.into_inner()
776        }
777    }
778}
779
780#[cfg(test)]
781#[path = "application_heartbeat_tests.rs"]
782mod tests;