Skip to main content

open_net/api/ws/
application_heartbeat.rs

1//! Application-level WebSocket heartbeat support.
2//!
3//! WebSocket control Ping/Pong frames are handled by the transport heartbeat.
4//! This module provides a separate, bounded loop for exchanges that require a
5//! text or binary heartbeat message at the application protocol level.
6//!
7//! These are ordinary data messages: they share application queue admission and
8//! wait for an already-started fragmented message to finish. They do not have
9//! the interleaving privileges of RFC 6455 Ping/Pong control frames. If the
10//! application protocol needs scheduling opportunities within a large payload,
11//! it must define multiple complete messages or use a separate connection; SDK
12//! frame fragmentation preserves the original logical message.
13
14use super::{
15    ConnectionId, DisconnectedPolicy, IncomingMessage, Message, MessageReceiver, ReceiveOptions,
16    ReceiveOverflow, SendOptions, Sender,
17};
18use crate::error::{ErrorKind, ErrorStage, ReceiveError};
19use crate::module::ws_client::{session_runtime::SessionRuntime, ws_client_inner::WSClientInner};
20use crate::{NetError, Result};
21use std::panic::{catch_unwind, AssertUnwindSafe};
22use std::sync::{Arc, Mutex, MutexGuard, Weak};
23use std::time::{Duration, Instant as StdInstant, SystemTime};
24use tokio::task::JoinHandle;
25use tokio::time::{self, Instant};
26use tokio_util::sync::CancellationToken;
27
28/// Callback used to recognize an application-level heartbeat response.
29///
30/// The callback must only inspect the immutable message. A callback panic is
31/// converted to [`ErrorKind::CallbackPanicked`] and never escapes the task.
32/// Matching is message-level and connection-scoped; protocols that can deliver
33/// delayed or repeated responses should include a request nonce in the payload
34/// and verify it in this callback.
35pub type ApplicationHeartbeatMatcher =
36    Arc<dyn Fn(&IncomingMessage) -> bool + Send + Sync + 'static>;
37
38/// Action to take after an application heartbeat send or response failure.
39#[derive(Clone, Copy, Debug, Eq, PartialEq)]
40pub enum ApplicationHeartbeatFailureAction {
41    /// Record the failure and continue with the next scheduled heartbeat.
42    Continue,
43    /// Record the failure and stop the application heartbeat loop.
44    Stop,
45}
46
47/// Configuration for a fixed-interval application heartbeat loop.
48#[derive(Clone)]
49pub struct ApplicationHeartbeatConfig {
50    /// Delay between heartbeat attempts. The first heartbeat is sent after this delay.
51    pub interval: Duration,
52    /// Text or binary payload sent on every heartbeat attempt.
53    pub payload: Message,
54    /// Maximum time allowed for queue admission and the local write.
55    pub send_timeout: Duration,
56    /// How long to wait for a matching response after a successful write.
57    /// `None` disables response matching and timeout accounting.
58    pub response_timeout: Option<Duration>,
59    /// Number of consecutive response timeouts that trigger
60    /// `Stop` or a configured heartbeat reconnect.
61    pub max_missed_responses: u32,
62    /// Optional application response matcher. When absent, writes are observed only.
63    pub matcher: Option<ApplicationHeartbeatMatcher>,
64    /// Receive queue used exclusively by the heartbeat loop when a matcher is set.
65    /// The default is [`ReceiveOverflow::DropOldest`] so unrelated business
66    /// traffic cannot close the session; selecting `Disconnect` makes queue
67    /// overflow a heartbeat receiver failure by design.
68    pub receive_options: ReceiveOptions,
69    /// Failure handling policy for writes, receiver failures, and response timeouts.
70    pub failure_action: ApplicationHeartbeatFailureAction,
71}
72
73impl ApplicationHeartbeatConfig {
74    /// Creates a fixed-interval heartbeat configuration with no response matching.
75    pub fn new(interval: Duration, payload: impl Into<Message>) -> Self {
76        Self {
77            interval,
78            payload: payload.into(),
79            send_timeout: Duration::from_secs(10),
80            response_timeout: None,
81            max_missed_responses: 1,
82            matcher: None,
83            receive_options: ReceiveOptions {
84                overflow: ReceiveOverflow::DropOldest,
85                ..ReceiveOptions::default()
86            },
87            failure_action: ApplicationHeartbeatFailureAction::Continue,
88        }
89    }
90
91    /// Sets the timeout used for queue admission and local WebSocket writing.
92    pub fn with_send_timeout(mut self, timeout: Duration) -> Self {
93        self.send_timeout = timeout;
94        self
95    }
96
97    /// Enables response matching with the supplied timeout and matcher.
98    pub fn with_response_matcher(
99        mut self,
100        timeout: Duration,
101        matcher: impl Fn(&IncomingMessage) -> bool + Send + Sync + 'static,
102    ) -> Self {
103        self.response_timeout = Some(timeout);
104        self.matcher = Some(Arc::new(matcher));
105        self
106    }
107
108    /// Sets the maximum number of consecutive response timeouts.
109    pub fn with_max_missed_responses(mut self, max_missed_responses: u32) -> Self {
110        self.max_missed_responses = max_missed_responses;
111        self
112    }
113
114    /// Sets the receive queue options used by the heartbeat loop.
115    pub fn with_receive_options(mut self, receive_options: ReceiveOptions) -> Self {
116        self.receive_options = receive_options;
117        self
118    }
119
120    /// Sets whether failures stop or leave the loop running.
121    pub fn with_failure_action(
122        mut self,
123        failure_action: ApplicationHeartbeatFailureAction,
124    ) -> Self {
125        self.failure_action = failure_action;
126        self
127    }
128
129    /// Validates values that would otherwise create a busy loop or an unbounded wait.
130    pub fn validate(&self) -> Result<()> {
131        if self.interval.is_zero() {
132            return Err(NetError::config(
133                "application_heartbeat.interval",
134                "must be greater than zero",
135            ));
136        }
137        if Instant::now().checked_add(self.interval).is_none() {
138            return Err(NetError::config(
139                "application_heartbeat.interval",
140                "cannot be represented as a timer deadline",
141            ));
142        }
143        if self.send_timeout.is_zero() {
144            return Err(NetError::config(
145                "application_heartbeat.send_timeout",
146                "must be greater than zero",
147            ));
148        }
149        if StdInstant::now().checked_add(self.send_timeout).is_none() {
150            return Err(NetError::config(
151                "application_heartbeat.send_timeout",
152                "cannot be represented as a deadline",
153            ));
154        }
155        if self.max_missed_responses == 0 {
156            return Err(NetError::config(
157                "application_heartbeat.max_missed_responses",
158                "must be greater than zero",
159            ));
160        }
161        if self.matcher.is_some() && self.receive_options.include_control_frames {
162            return Err(NetError::config(
163                "application_heartbeat.receive_options.include_control_frames",
164                "must be false so control Pong frames cannot acknowledge application heartbeats",
165            ));
166        }
167        if self.matcher.is_some() && self.response_timeout.is_none() {
168            return Err(NetError::config(
169                "application_heartbeat.response_timeout",
170                "is required when matcher is set",
171            ));
172        }
173        if let Some(timeout) = self.response_timeout {
174            if StdInstant::now().checked_add(timeout).is_none() {
175                return Err(NetError::config(
176                    "application_heartbeat.response_timeout",
177                    "must fit an absolute deadline",
178                ));
179            }
180            if timeout.is_zero() {
181                return Err(NetError::config(
182                    "application_heartbeat.response_timeout",
183                    "must be greater than zero",
184                ));
185            }
186            if Instant::now().checked_add(timeout).is_none() {
187                return Err(NetError::config(
188                    "application_heartbeat.response_timeout",
189                    "cannot be represented as a deadline",
190                ));
191            }
192            if self.matcher.is_none() {
193                return Err(NetError::config(
194                    "application_heartbeat.matcher",
195                    "is required when response_timeout is set",
196                ));
197            }
198        }
199        if self.matcher.is_some() {
200            self.receive_options.validate()?;
201        }
202        Ok(())
203    }
204}
205
206impl std::fmt::Debug for ApplicationHeartbeatConfig {
207    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
208        formatter
209            .debug_struct("ApplicationHeartbeatConfig")
210            .field("interval", &self.interval)
211            .field("payload_bytes", &self.payload.len())
212            .field("send_timeout", &self.send_timeout)
213            .field("response_timeout", &self.response_timeout)
214            .field("max_missed_responses", &self.max_missed_responses)
215            .field("has_matcher", &self.matcher.is_some())
216            .field("receive_options", &self.receive_options)
217            .field("failure_action", &self.failure_action)
218            .finish()
219    }
220}
221
222/// Current state of an application heartbeat loop.
223#[derive(Clone, Copy, Debug, Eq, PartialEq)]
224pub enum ApplicationHeartbeatState {
225    /// The loop is waiting for its next scheduled send.
226    Running,
227    /// A heartbeat was written and a response is being matched.
228    WaitingForResponse,
229    /// The loop has stopped because its handle was stopped, its session ended, or policy stopped it.
230    Stopped,
231}
232
233/// Immutable counters and last-error classification for an application heartbeat.
234#[derive(Clone, Copy, Debug, Eq, PartialEq)]
235pub struct ApplicationHeartbeatSnapshot {
236    /// Current loop state.
237    pub state: ApplicationHeartbeatState,
238    /// Number of successful local writes.
239    pub sent: u64,
240    /// Number of matched application responses.
241    pub matched_responses: u64,
242    /// Number of response deadlines reached without a match.
243    pub missed_responses: u64,
244    /// Number of send, receive, matcher, or response failures.
245    pub failures: u64,
246    /// Classification of the most recent failure, if any.
247    pub last_error: Option<ErrorKind>,
248}
249
250impl Default for ApplicationHeartbeatSnapshot {
251    fn default() -> Self {
252        Self {
253            state: ApplicationHeartbeatState::Running,
254            sent: 0,
255            matched_responses: 0,
256            missed_responses: 0,
257            failures: 0,
258            last_error: None,
259        }
260    }
261}
262
263/// Controls one application heartbeat loop.
264///
265/// Dropping the handle requests cancellation and aborts only the heartbeat task;
266/// the owning WebSocket session remains alive.
267pub struct ApplicationHeartbeatHandle {
268    cancellation: CancellationToken,
269    snapshot: Arc<Mutex<ApplicationHeartbeatSnapshot>>,
270    task: Option<JoinHandle<()>>,
271}
272
273impl ApplicationHeartbeatHandle {
274    pub(crate) fn new(
275        sender: Sender,
276        receiver: Option<MessageReceiver>,
277        config: ApplicationHeartbeatConfig,
278        reconnect: Option<Weak<WSClientInner>>,
279    ) -> Result<Self> {
280        config.validate()?;
281        let cancellation = CancellationToken::new();
282        let snapshot = Arc::new(Mutex::new(ApplicationHeartbeatSnapshot::default()));
283        let runtime = sender.runtime.clone();
284        let task = runtime.clone().spawn(run_heartbeat(
285            sender,
286            receiver,
287            config,
288            cancellation.clone(),
289            snapshot.clone(),
290            runtime,
291            reconnect,
292        ))?;
293        Ok(Self {
294            cancellation,
295            snapshot,
296            task: Some(task),
297        })
298    }
299
300    /// Requests the loop to stop. The request is idempotent.
301    pub fn stop(&self) {
302        self.cancellation.cancel();
303    }
304
305    /// Returns a point-in-time snapshot without waiting for the loop.
306    pub fn snapshot(&self) -> ApplicationHeartbeatSnapshot {
307        snapshot_copy(&self.snapshot)
308    }
309
310    /// Waits for the loop task to finish and returns its final snapshot.
311    pub async fn stopped(mut self) -> Result<ApplicationHeartbeatSnapshot> {
312        let task = self
313            .task
314            .take()
315            .ok_or_else(|| NetError::from(ErrorKind::Internal).with_stage(ErrorStage::Runtime))?;
316        task.await.map_err(|error| {
317            NetError::with_source(ErrorKind::EngineDropped, error).with_stage(ErrorStage::Runtime)
318        })?;
319        Ok(snapshot_copy(&self.snapshot))
320    }
321}
322
323impl Drop for ApplicationHeartbeatHandle {
324    fn drop(&mut self) {
325        self.cancellation.cancel();
326        if let Some(task) = self.task.take() {
327            task.abort();
328        }
329    }
330}
331
332#[derive(Clone, Copy, Debug, Eq, PartialEq)]
333enum ResponseWait {
334    Matched,
335    TimedOut,
336    Closed,
337    Cancelled,
338    Failed(HeartbeatFailure),
339}
340
341#[derive(Clone, Copy, Debug, Eq, PartialEq)]
342enum HeartbeatPhase {
343    SendOptions,
344    Send,
345    Connection,
346    ResponseWait,
347    Matcher,
348    Receive,
349}
350
351impl HeartbeatPhase {
352    fn name(self) -> &'static str {
353        match self {
354            Self::SendOptions => "send_options",
355            Self::Send => "send",
356            Self::Connection => "connection",
357            Self::ResponseWait => "response_wait",
358            Self::Matcher => "matcher",
359            Self::Receive => "receive",
360        }
361    }
362}
363
364#[derive(Clone, Copy, Debug, Eq, PartialEq)]
365struct HeartbeatFailure {
366    phase: HeartbeatPhase,
367    kind: ErrorKind,
368    stage: Option<ErrorStage>,
369}
370
371const HEARTBEAT_STAGE_SLOTS: usize = ErrorStage::Runtime as usize + 2;
372
373#[derive(Default)]
374struct HeartbeatDiagnostics {
375    client_id: u64,
376    last: Option<HeartbeatFailure>,
377    failures: u64,
378    // One bit per SDK ErrorKind for each fixed phase/stage, including no stage.
379    // None of these slots are keyed by user-controlled data.
380    reported: [[u64; HEARTBEAT_STAGE_SLOTS]; 6],
381}
382
383impl HeartbeatDiagnostics {
384    fn should_report(&mut self, failure: HeartbeatFailure, enabled: bool) -> bool {
385        if matches!(failure.kind, ErrorKind::Cancelled | ErrorKind::Closed) {
386            self.last = None;
387            self.failures = 0;
388            self.reported = [[0; HEARTBEAT_STAGE_SLOTS]; 6];
389            return false;
390        }
391        self.last = Some(failure);
392        self.failures = self.failures.saturating_add(1);
393        if !enabled {
394            // An unobserved failure does not spend a later subscriber's first
395            // diagnostic. Queue rejection retains the logger's drop semantics.
396            self.reported = [[0; HEARTBEAT_STAGE_SLOTS]; 6];
397            return false;
398        }
399        // ErrorKind currently has 40 data-free variants. Keep future out-of-range
400        // classifications bounded as well, without risking a diagnostic panic.
401        let bit = 1_u64
402            .checked_shl(failure.kind as u32)
403            .unwrap_or(1_u64 << 63);
404        let stage = failure
405            .stage
406            .map_or(0, |stage| stage as usize + 1)
407            .min(HEARTBEAT_STAGE_SLOTS - 1);
408        let reported = &mut self.reported[failure.phase as usize][stage];
409        let first = *reported & bit == 0;
410        *reported |= bit;
411        first
412    }
413
414    fn failed(&mut self, session_id: u64, failure: HeartbeatFailure) {
415        if self.should_report(
416            failure,
417            crate::OpenNetLogger::is_enabled_for(crate::LogType::WSC, crate::LogLevel::Error),
418        ) {
419            crate::log_e!(crate::LogType::WSC; "ws.heartbeat.failed",
420                "client_id|session_id|phase|kind|stage|failures", self.client_id, session_id, failure.phase.name(),
421                format!("{:?}", failure.kind), failure.stage.map(|stage| format!("{stage:?}")), self.failures);
422        }
423    }
424
425    fn error(&mut self, session_id: u64, phase: HeartbeatPhase, error: &NetError) {
426        self.failed(
427            session_id,
428            HeartbeatFailure {
429                phase,
430                kind: error.kind(),
431                stage: error.context().stage,
432            },
433        );
434    }
435
436    fn recovered(&mut self, session_id: u64) {
437        if let Some(failure) = self.last.take() {
438            crate::log_s!(crate::LogType::WSC, crate::LogLevel::Key; "ws.heartbeat.recovered",
439                "client_id|session_id|phase|failures", self.client_id, session_id, failure.phase.name(), self.failures);
440        }
441        self.failures = 0;
442        self.reported = [[0; HEARTBEAT_STAGE_SLOTS]; 6];
443    }
444
445    fn stopped(&self, session_id: u64) {
446        if let Some(failure) = self.last {
447            crate::log_s!(crate::LogType::WSC, crate::LogLevel::Key; "ws.heartbeat.stopped",
448                "client_id|session_id|phase|kind|failures", self.client_id, session_id, failure.phase.name(), format!("{:?}", failure.kind), self.failures);
449        }
450    }
451}
452
453async fn run_heartbeat(
454    sender: Sender,
455    mut receiver: Option<MessageReceiver>,
456    config: ApplicationHeartbeatConfig,
457    cancellation: CancellationToken,
458    snapshot: Arc<Mutex<ApplicationHeartbeatSnapshot>>,
459    runtime: Arc<SessionRuntime>,
460    reconnect: Option<Weak<WSClientInner>>,
461) {
462    let now = Instant::now();
463    let first_tick = match now.checked_add(config.interval) {
464        Some(deadline) => deadline,
465        None => now,
466    };
467    let mut ticker = time::interval_at(first_tick, config.interval);
468    ticker.set_missed_tick_behavior(time::MissedTickBehavior::Skip);
469    let mut missed_streak = MissedResponses::default();
470    let mut stopped_by_policy = false;
471    let session_closed = runtime.closed_token();
472    let session_id = runtime.id.as_u64();
473    let mut diagnostics = HeartbeatDiagnostics {
474        client_id: runtime.client_id.as_u64(),
475        ..Default::default()
476    };
477
478    'heartbeat: loop {
479        tokio::select! {
480            _ = cancellation.cancelled() => break 'heartbeat,
481            _ = session_closed.cancelled() => break 'heartbeat,
482            _ = ticker.tick() => {
483                snapshot_update(&snapshot, |current| {
484                    current.state = ApplicationHeartbeatState::Running;
485                });
486                let options = match send_options(config.send_timeout) {
487                    Ok(options) => options,
488                    Err(error) => {
489                        diagnostics.error(session_id, HeartbeatPhase::SendOptions, &error);
490                        // A configuration/deadline failure is not a missed
491                        // response. Keep the response-timeout streak scoped to
492                        // actual response deadlines.
493                        missed_streak.count = 0;
494                        let should_stop = record_failure(
495                            &snapshot,
496                            &error,
497                            config.failure_action,
498                        );
499                        if should_stop {
500                            stopped_by_policy = true;
501                            break 'heartbeat;
502                        }
503                        continue;
504                    }
505                };
506                let sent_at = SystemTime::now();
507                let builder = sender.message(config.payload.clone()).options(options);
508                let send = async {
509                    if reconnect.is_some() {
510                        let receipt = builder.enqueue().await.map_err(crate::EnqueueError::into_error)?;
511                        receipt.written_connection().await.map(Some)
512                    } else {
513                        builder.send().await.map(|_| None)
514                    }
515                };
516                let send_result = tokio::select! {
517                    _ = cancellation.cancelled() => break 'heartbeat,
518                    _ = session_closed.cancelled() => break 'heartbeat,
519                    result = send => result,
520                };
521                let written_connection = match send_result {
522                    Ok(connection) => connection,
523                    Err(error) => {
524                        diagnostics.error(session_id, HeartbeatPhase::Send, &error);
525                        missed_streak.count = 0;
526                        if record_failure(&snapshot, &error, config.failure_action) {
527                            stopped_by_policy = true;
528                            break 'heartbeat;
529                        }
530                        continue;
531                    }
532                };
533                snapshot_update(&snapshot, |current| {
534                    increment(&mut current.sent);
535                });
536                let (Some(matcher), Some(receiver)) =
537                    (config.matcher.as_ref(), receiver.as_mut())
538                else {
539                    diagnostics.recovered(session_id);
540                    continue;
541                };
542                // The recovery mode uses the writer's ID even if another connection is
543                // already active. The original entry point keeps its lifecycle checks.
544                let connection = match written_connection {
545                    Some(connection) => Ok(Some(connection)),
546                    None => runtime.current_connection_id(),
547                };
548                let connection_id = match connection {
549                    Ok(Some(connection)) => connection,
550                    other => {
551                        let error = match other {
552                            Err(error) => {
553                                diagnostics.error(session_id, HeartbeatPhase::Connection, &error);
554                                error
555                            }
556                            Ok(_) => {
557                                // A retired writer is expected. Only a policy stop needs
558                                // to identify it as the last diagnostic.
559                                if config.failure_action == ApplicationHeartbeatFailureAction::Stop {
560                                    diagnostics.last = Some(HeartbeatFailure {
561                                        phase: HeartbeatPhase::Connection,
562                                        kind: ErrorKind::NotConnected,
563                                        stage: Some(ErrorStage::Runtime),
564                                    });
565                                    diagnostics.failures = diagnostics.failures.saturating_add(1);
566                                }
567                                NetError::from(ErrorKind::NotConnected).with_stage(ErrorStage::Runtime)
568                            }
569                        };
570                        missed_streak.count = 0;
571                        if record_failure(&snapshot, &error, config.failure_action) {
572                            stopped_by_policy = true;
573                            break 'heartbeat;
574                        }
575                        continue;
576                    }
577                };
578                snapshot_update(&snapshot, |current| {
579                    current.state = ApplicationHeartbeatState::WaitingForResponse;
580                });
581                let timeout = match config.response_timeout {
582                    Some(timeout) => timeout,
583                    None => continue,
584                };
585                match wait_for_response(
586                    receiver,
587                    matcher,
588                    timeout,
589                    &cancellation,
590                    &session_closed,
591                    connection_id,
592                    sent_at,
593                ).await {
594                    ResponseWait::Matched => {
595                        missed_streak.count = 0;
596                        snapshot_update(&snapshot, |current| {
597                            increment(&mut current.matched_responses);
598                            current.state = ApplicationHeartbeatState::Running;
599                        });
600                        diagnostics.recovered(session_id);
601                    }
602                    ResponseWait::TimedOut => {
603                        diagnostics.failed(session_id, HeartbeatFailure {
604                            phase: HeartbeatPhase::ResponseWait, kind: ErrorKind::TimedOut, stage: Some(ErrorStage::Receive),
605                        });
606                        let consecutive_missed = missed_streak.record(written_connection);
607                        snapshot_update(&snapshot, |current| {
608                            increment(&mut current.missed_responses);
609                            current.last_error = Some(ErrorKind::TimedOut);
610                            increment(&mut current.failures);
611                            current.state = ApplicationHeartbeatState::Running;
612                        });
613                        if consecutive_missed < config.max_missed_responses {
614                            continue;
615                        }
616                        if let Some(client) = &reconnect {
617                            missed_streak.count = 0;
618                            let result = tokio::select! {
619                                biased;
620                                _ = cancellation.cancelled() => break 'heartbeat,
621                                _ = session_closed.cancelled() => break 'heartbeat,
622                                result = recover_connection(client, &runtime, connection_id, &cancellation) => result,
623                            };
624                            if let Err(error) = result {
625                                if runtime.is_closed() || matches!(error.kind(), ErrorKind::Cancelled | ErrorKind::Closed | ErrorKind::EngineDropped) {
626                                    break 'heartbeat;
627                                }
628                                diagnostics.error(session_id, HeartbeatPhase::Connection, &error);
629                                if record_failure(&snapshot, &error, config.failure_action) {
630                                    stopped_by_policy = true;
631                                    break 'heartbeat;
632                                }
633                            }
634                        } else if config.failure_action == ApplicationHeartbeatFailureAction::Stop {
635                            stopped_by_policy = true;
636                            break 'heartbeat;
637                        }
638                    }
639                    ResponseWait::Closed | ResponseWait::Cancelled => break 'heartbeat,
640                    ResponseWait::Failed(failure) => {
641                        diagnostics.failed(session_id, failure);
642                        // Receiver and matcher failures are distinct from a
643                        // response deadline, so they reset consecutive misses.
644                        missed_streak.count = 0;
645                        let should_stop = record_failure_kind(
646                            &snapshot,
647                            failure.kind,
648                            config.failure_action,
649                        );
650                        if should_stop {
651                            stopped_by_policy = true;
652                            break 'heartbeat;
653                        }
654                    }
655                }
656            }
657        }
658    }
659    if let Some(receiver) = receiver.as_ref() {
660        receiver.unsubscribe();
661    }
662    snapshot_update(&snapshot, |current| {
663        current.state = ApplicationHeartbeatState::Stopped;
664    });
665    if stopped_by_policy {
666        diagnostics.stopped(session_id);
667    }
668}
669
670#[derive(Default)]
671struct MissedResponses {
672    connection: Option<ConnectionId>,
673    count: u32,
674}
675
676impl MissedResponses {
677    /// The legacy policy uses a session-wide streak. Recovery supplies the
678    /// connection from the write receipt and starts each writer at zero.
679    fn record(&mut self, connection: Option<ConnectionId>) -> u32 {
680        if self.connection != connection {
681            self.connection = connection;
682            self.count = 0;
683        }
684        self.count = self.count.saturating_add(1);
685        self.count
686    }
687}
688
689async fn recover_connection(
690    client: &Weak<WSClientInner>,
691    runtime: &Arc<SessionRuntime>,
692    connection: ConnectionId,
693    cancellation: &CancellationToken,
694) -> Result<()> {
695    client
696        .upgrade()
697        .ok_or_else(|| NetError::from(ErrorKind::Closed))?
698        .reconnect_if_current(runtime, connection, Some(cancellation.clone()))
699        .await?;
700    runtime
701        .lifecycle
702        .wait_connected_after(Some(connection))
703        .await?;
704    Ok(())
705}
706
707async fn wait_for_response(
708    receiver: &mut MessageReceiver,
709    matcher: &ApplicationHeartbeatMatcher,
710    timeout: Duration,
711    cancellation: &CancellationToken,
712    session_closed: &CancellationToken,
713    connection_id: ConnectionId,
714    sent_at: SystemTime,
715) -> ResponseWait {
716    let deadline = time::sleep(timeout);
717    tokio::pin!(deadline);
718    loop {
719        tokio::select! {
720            _ = cancellation.cancelled() => return ResponseWait::Cancelled,
721            _ = session_closed.cancelled() => return ResponseWait::Closed,
722            _ = &mut deadline => return ResponseWait::TimedOut,
723            result = receiver.recv() => {
724                match result {
725                    Ok(Some(message)) => {
726                        if connection_id != message.connection_id() {
727                            continue;
728                        }
729                        if message.received_at().duration_since(sent_at).is_err() {
730                            continue;
731                        }
732                        let matches = match catch_unwind(AssertUnwindSafe(|| matcher(&message))) {
733                            Ok(matches) => matches,
734                            Err(_) => return ResponseWait::Failed(HeartbeatFailure {
735                                phase: HeartbeatPhase::Matcher, kind: ErrorKind::CallbackPanicked, stage: Some(ErrorStage::Receive),
736                            }),
737                        };
738                        if matches {
739                            return ResponseWait::Matched;
740                        }
741                    }
742                    Ok(None) => return ResponseWait::Closed,
743                    // DropOldest is the safe default for a matcher queue. A
744                    // lag means unrelated messages were discarded; keep
745                    // waiting for the current connection's matching response
746                    // instead of turning normal market-data pressure into a
747                    // socket failure.
748                    Err(ReceiveError::Lagged { .. }) => continue,
749                    Err(ReceiveError::Failed(error)) => {
750                        return ResponseWait::Failed(HeartbeatFailure {
751                            phase: HeartbeatPhase::Receive, kind: error.kind(), stage: error.context().stage,
752                        });
753                    }
754                }
755            }
756        }
757    }
758}
759
760fn send_options(timeout: Duration) -> Result<SendOptions> {
761    let deadline = StdInstant::now().checked_add(timeout).ok_or_else(|| {
762        NetError::config(
763            "application_heartbeat.send_timeout",
764            "cannot be represented as a deadline",
765        )
766    })?;
767    Ok(SendOptions {
768        disconnected: DisconnectedPolicy::WaitForReconnect,
769        enqueue_timeout: Some(timeout),
770        write_timeout: timeout,
771        deadline: Some(deadline),
772        ..SendOptions::default()
773    })
774}
775
776fn record_failure(
777    snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
778    error: &NetError,
779    action: ApplicationHeartbeatFailureAction,
780) -> bool {
781    record_failure_kind(snapshot, error.kind(), action)
782}
783
784fn record_failure_kind(
785    snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
786    kind: ErrorKind,
787    action: ApplicationHeartbeatFailureAction,
788) -> bool {
789    snapshot_update(snapshot, |current| {
790        increment(&mut current.failures);
791        current.last_error = Some(kind);
792        current.state = ApplicationHeartbeatState::Running;
793    });
794    action == ApplicationHeartbeatFailureAction::Stop
795}
796
797fn increment(value: &mut u64) {
798    if let Some(next) = value.checked_add(1) {
799        *value = next;
800    }
801}
802
803fn snapshot_copy(
804    snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
805) -> ApplicationHeartbeatSnapshot {
806    *lock_snapshot(snapshot)
807}
808
809fn snapshot_update<F>(snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>, update: F)
810where
811    F: FnOnce(&mut ApplicationHeartbeatSnapshot),
812{
813    let mut current = lock_snapshot(snapshot);
814    update(&mut current);
815}
816
817fn lock_snapshot<'a>(
818    snapshot: &'a Arc<Mutex<ApplicationHeartbeatSnapshot>>,
819) -> MutexGuard<'a, ApplicationHeartbeatSnapshot> {
820    match snapshot.lock() {
821        Ok(guard) => guard,
822        Err(poisoned) => {
823            crate::log_e!(crate::common::log::log_def::LogType::WSC;
824                "ws.heartbeat.lock_snapshot", "arg_snapshot_present|error", true, "lock_poisoned_recovered");
825            poisoned.into_inner()
826        }
827    }
828}
829
830#[cfg(test)]
831#[path = "application_heartbeat_tests.rs"]
832mod tests;