1use super::{
15 ConnectionId, DisconnectedPolicy, IncomingMessage, Message, MessageReceiver, ReceiveOptions,
16 ReceiveOverflow, SendOptions, Sender,
17};
18use crate::error::{ErrorKind, ErrorStage, ReceiveError};
19use crate::module::ws_client::{session_runtime::SessionRuntime, ws_client_inner::WSClientInner};
20use crate::{NetError, Result};
21use std::panic::{catch_unwind, AssertUnwindSafe};
22use std::sync::{Arc, Mutex, MutexGuard, Weak};
23use std::time::{Duration, Instant as StdInstant, SystemTime};
24use tokio::task::JoinHandle;
25use tokio::time::{self, Instant};
26use tokio_util::sync::CancellationToken;
27
28pub type ApplicationHeartbeatMatcher =
36 Arc<dyn Fn(&IncomingMessage) -> bool + Send + Sync + 'static>;
37
38#[derive(Clone, Copy, Debug, Eq, PartialEq)]
40pub enum ApplicationHeartbeatFailureAction {
41 Continue,
43 Stop,
45}
46
47#[derive(Clone)]
49pub struct ApplicationHeartbeatConfig {
50 pub interval: Duration,
52 pub payload: Message,
54 pub send_timeout: Duration,
56 pub response_timeout: Option<Duration>,
59 pub max_missed_responses: u32,
62 pub matcher: Option<ApplicationHeartbeatMatcher>,
64 pub receive_options: ReceiveOptions,
69 pub failure_action: ApplicationHeartbeatFailureAction,
71}
72
73impl ApplicationHeartbeatConfig {
74 pub fn new(interval: Duration, payload: impl Into<Message>) -> Self {
76 Self {
77 interval,
78 payload: payload.into(),
79 send_timeout: Duration::from_secs(10),
80 response_timeout: None,
81 max_missed_responses: 1,
82 matcher: None,
83 receive_options: ReceiveOptions {
84 overflow: ReceiveOverflow::DropOldest,
85 ..ReceiveOptions::default()
86 },
87 failure_action: ApplicationHeartbeatFailureAction::Continue,
88 }
89 }
90
91 pub fn with_send_timeout(mut self, timeout: Duration) -> Self {
93 self.send_timeout = timeout;
94 self
95 }
96
97 pub fn with_response_matcher(
99 mut self,
100 timeout: Duration,
101 matcher: impl Fn(&IncomingMessage) -> bool + Send + Sync + 'static,
102 ) -> Self {
103 self.response_timeout = Some(timeout);
104 self.matcher = Some(Arc::new(matcher));
105 self
106 }
107
108 pub fn with_max_missed_responses(mut self, max_missed_responses: u32) -> Self {
110 self.max_missed_responses = max_missed_responses;
111 self
112 }
113
114 pub fn with_receive_options(mut self, receive_options: ReceiveOptions) -> Self {
116 self.receive_options = receive_options;
117 self
118 }
119
120 pub fn with_failure_action(
122 mut self,
123 failure_action: ApplicationHeartbeatFailureAction,
124 ) -> Self {
125 self.failure_action = failure_action;
126 self
127 }
128
129 pub fn validate(&self) -> Result<()> {
131 if self.interval.is_zero() {
132 return Err(NetError::config(
133 "application_heartbeat.interval",
134 "must be greater than zero",
135 ));
136 }
137 if Instant::now().checked_add(self.interval).is_none() {
138 return Err(NetError::config(
139 "application_heartbeat.interval",
140 "cannot be represented as a timer deadline",
141 ));
142 }
143 if self.send_timeout.is_zero() {
144 return Err(NetError::config(
145 "application_heartbeat.send_timeout",
146 "must be greater than zero",
147 ));
148 }
149 if StdInstant::now().checked_add(self.send_timeout).is_none() {
150 return Err(NetError::config(
151 "application_heartbeat.send_timeout",
152 "cannot be represented as a deadline",
153 ));
154 }
155 if self.max_missed_responses == 0 {
156 return Err(NetError::config(
157 "application_heartbeat.max_missed_responses",
158 "must be greater than zero",
159 ));
160 }
161 if self.matcher.is_some() && self.receive_options.include_control_frames {
162 return Err(NetError::config(
163 "application_heartbeat.receive_options.include_control_frames",
164 "must be false so control Pong frames cannot acknowledge application heartbeats",
165 ));
166 }
167 if self.matcher.is_some() && self.response_timeout.is_none() {
168 return Err(NetError::config(
169 "application_heartbeat.response_timeout",
170 "is required when matcher is set",
171 ));
172 }
173 if let Some(timeout) = self.response_timeout {
174 if StdInstant::now().checked_add(timeout).is_none() {
175 return Err(NetError::config(
176 "application_heartbeat.response_timeout",
177 "must fit an absolute deadline",
178 ));
179 }
180 if timeout.is_zero() {
181 return Err(NetError::config(
182 "application_heartbeat.response_timeout",
183 "must be greater than zero",
184 ));
185 }
186 if Instant::now().checked_add(timeout).is_none() {
187 return Err(NetError::config(
188 "application_heartbeat.response_timeout",
189 "cannot be represented as a deadline",
190 ));
191 }
192 if self.matcher.is_none() {
193 return Err(NetError::config(
194 "application_heartbeat.matcher",
195 "is required when response_timeout is set",
196 ));
197 }
198 }
199 if self.matcher.is_some() {
200 self.receive_options.validate()?;
201 }
202 Ok(())
203 }
204}
205
206impl std::fmt::Debug for ApplicationHeartbeatConfig {
207 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
208 formatter
209 .debug_struct("ApplicationHeartbeatConfig")
210 .field("interval", &self.interval)
211 .field("payload_bytes", &self.payload.len())
212 .field("send_timeout", &self.send_timeout)
213 .field("response_timeout", &self.response_timeout)
214 .field("max_missed_responses", &self.max_missed_responses)
215 .field("has_matcher", &self.matcher.is_some())
216 .field("receive_options", &self.receive_options)
217 .field("failure_action", &self.failure_action)
218 .finish()
219 }
220}
221
222#[derive(Clone, Copy, Debug, Eq, PartialEq)]
224pub enum ApplicationHeartbeatState {
225 Running,
227 WaitingForResponse,
229 Stopped,
231}
232
233#[derive(Clone, Copy, Debug, Eq, PartialEq)]
235pub struct ApplicationHeartbeatSnapshot {
236 pub state: ApplicationHeartbeatState,
238 pub sent: u64,
240 pub matched_responses: u64,
242 pub missed_responses: u64,
244 pub failures: u64,
246 pub last_error: Option<ErrorKind>,
248}
249
250impl Default for ApplicationHeartbeatSnapshot {
251 fn default() -> Self {
252 Self {
253 state: ApplicationHeartbeatState::Running,
254 sent: 0,
255 matched_responses: 0,
256 missed_responses: 0,
257 failures: 0,
258 last_error: None,
259 }
260 }
261}
262
263pub struct ApplicationHeartbeatHandle {
268 cancellation: CancellationToken,
269 snapshot: Arc<Mutex<ApplicationHeartbeatSnapshot>>,
270 task: Option<JoinHandle<()>>,
271}
272
273impl ApplicationHeartbeatHandle {
274 pub(crate) fn new(
275 sender: Sender,
276 receiver: Option<MessageReceiver>,
277 config: ApplicationHeartbeatConfig,
278 reconnect: Option<Weak<WSClientInner>>,
279 ) -> Result<Self> {
280 config.validate()?;
281 let cancellation = CancellationToken::new();
282 let snapshot = Arc::new(Mutex::new(ApplicationHeartbeatSnapshot::default()));
283 let runtime = sender.runtime.clone();
284 let task = runtime.clone().spawn(run_heartbeat(
285 sender,
286 receiver,
287 config,
288 cancellation.clone(),
289 snapshot.clone(),
290 runtime,
291 reconnect,
292 ))?;
293 Ok(Self {
294 cancellation,
295 snapshot,
296 task: Some(task),
297 })
298 }
299
300 pub fn stop(&self) {
302 self.cancellation.cancel();
303 }
304
305 pub fn snapshot(&self) -> ApplicationHeartbeatSnapshot {
307 snapshot_copy(&self.snapshot)
308 }
309
310 pub async fn stopped(mut self) -> Result<ApplicationHeartbeatSnapshot> {
312 let task = self
313 .task
314 .take()
315 .ok_or_else(|| NetError::from(ErrorKind::Internal).with_stage(ErrorStage::Runtime))?;
316 task.await.map_err(|error| {
317 NetError::with_source(ErrorKind::EngineDropped, error).with_stage(ErrorStage::Runtime)
318 })?;
319 Ok(snapshot_copy(&self.snapshot))
320 }
321}
322
323impl Drop for ApplicationHeartbeatHandle {
324 fn drop(&mut self) {
325 self.cancellation.cancel();
326 if let Some(task) = self.task.take() {
327 task.abort();
328 }
329 }
330}
331
332#[derive(Clone, Copy, Debug, Eq, PartialEq)]
333enum ResponseWait {
334 Matched,
335 TimedOut,
336 Closed,
337 Cancelled,
338 Failed(HeartbeatFailure),
339}
340
341#[derive(Clone, Copy, Debug, Eq, PartialEq)]
342enum HeartbeatPhase {
343 SendOptions,
344 Send,
345 Connection,
346 ResponseWait,
347 Matcher,
348 Receive,
349}
350
351impl HeartbeatPhase {
352 fn name(self) -> &'static str {
353 match self {
354 Self::SendOptions => "send_options",
355 Self::Send => "send",
356 Self::Connection => "connection",
357 Self::ResponseWait => "response_wait",
358 Self::Matcher => "matcher",
359 Self::Receive => "receive",
360 }
361 }
362}
363
364#[derive(Clone, Copy, Debug, Eq, PartialEq)]
365struct HeartbeatFailure {
366 phase: HeartbeatPhase,
367 kind: ErrorKind,
368 stage: Option<ErrorStage>,
369}
370
371const HEARTBEAT_STAGE_SLOTS: usize = ErrorStage::Runtime as usize + 2;
372
373#[derive(Default)]
374struct HeartbeatDiagnostics {
375 client_id: u64,
376 last: Option<HeartbeatFailure>,
377 failures: u64,
378 reported: [[u64; HEARTBEAT_STAGE_SLOTS]; 6],
381}
382
383impl HeartbeatDiagnostics {
384 fn should_report(&mut self, failure: HeartbeatFailure, enabled: bool) -> bool {
385 if matches!(failure.kind, ErrorKind::Cancelled | ErrorKind::Closed) {
386 self.last = None;
387 self.failures = 0;
388 self.reported = [[0; HEARTBEAT_STAGE_SLOTS]; 6];
389 return false;
390 }
391 self.last = Some(failure);
392 self.failures = self.failures.saturating_add(1);
393 if !enabled {
394 self.reported = [[0; HEARTBEAT_STAGE_SLOTS]; 6];
397 return false;
398 }
399 let bit = 1_u64
402 .checked_shl(failure.kind as u32)
403 .unwrap_or(1_u64 << 63);
404 let stage = failure
405 .stage
406 .map_or(0, |stage| stage as usize + 1)
407 .min(HEARTBEAT_STAGE_SLOTS - 1);
408 let reported = &mut self.reported[failure.phase as usize][stage];
409 let first = *reported & bit == 0;
410 *reported |= bit;
411 first
412 }
413
414 fn failed(&mut self, session_id: u64, failure: HeartbeatFailure) {
415 if self.should_report(
416 failure,
417 crate::OpenNetLogger::is_enabled_for(crate::LogType::WSC, crate::LogLevel::Error),
418 ) {
419 crate::log_e!(crate::LogType::WSC; "ws.heartbeat.failed",
420 "client_id|session_id|phase|kind|stage|failures", self.client_id, session_id, failure.phase.name(),
421 format!("{:?}", failure.kind), failure.stage.map(|stage| format!("{stage:?}")), self.failures);
422 }
423 }
424
425 fn error(&mut self, session_id: u64, phase: HeartbeatPhase, error: &NetError) {
426 self.failed(
427 session_id,
428 HeartbeatFailure {
429 phase,
430 kind: error.kind(),
431 stage: error.context().stage,
432 },
433 );
434 }
435
436 fn recovered(&mut self, session_id: u64) {
437 if let Some(failure) = self.last.take() {
438 crate::log_s!(crate::LogType::WSC, crate::LogLevel::Key; "ws.heartbeat.recovered",
439 "client_id|session_id|phase|failures", self.client_id, session_id, failure.phase.name(), self.failures);
440 }
441 self.failures = 0;
442 self.reported = [[0; HEARTBEAT_STAGE_SLOTS]; 6];
443 }
444
445 fn stopped(&self, session_id: u64) {
446 if let Some(failure) = self.last {
447 crate::log_s!(crate::LogType::WSC, crate::LogLevel::Key; "ws.heartbeat.stopped",
448 "client_id|session_id|phase|kind|failures", self.client_id, session_id, failure.phase.name(), format!("{:?}", failure.kind), self.failures);
449 }
450 }
451}
452
453async fn run_heartbeat(
454 sender: Sender,
455 mut receiver: Option<MessageReceiver>,
456 config: ApplicationHeartbeatConfig,
457 cancellation: CancellationToken,
458 snapshot: Arc<Mutex<ApplicationHeartbeatSnapshot>>,
459 runtime: Arc<SessionRuntime>,
460 reconnect: Option<Weak<WSClientInner>>,
461) {
462 let now = Instant::now();
463 let first_tick = match now.checked_add(config.interval) {
464 Some(deadline) => deadline,
465 None => now,
466 };
467 let mut ticker = time::interval_at(first_tick, config.interval);
468 ticker.set_missed_tick_behavior(time::MissedTickBehavior::Skip);
469 let mut missed_streak = MissedResponses::default();
470 let mut stopped_by_policy = false;
471 let session_closed = runtime.closed_token();
472 let session_id = runtime.id.as_u64();
473 let mut diagnostics = HeartbeatDiagnostics {
474 client_id: runtime.client_id.as_u64(),
475 ..Default::default()
476 };
477
478 'heartbeat: loop {
479 tokio::select! {
480 _ = cancellation.cancelled() => break 'heartbeat,
481 _ = session_closed.cancelled() => break 'heartbeat,
482 _ = ticker.tick() => {
483 snapshot_update(&snapshot, |current| {
484 current.state = ApplicationHeartbeatState::Running;
485 });
486 let options = match send_options(config.send_timeout) {
487 Ok(options) => options,
488 Err(error) => {
489 diagnostics.error(session_id, HeartbeatPhase::SendOptions, &error);
490 missed_streak.count = 0;
494 let should_stop = record_failure(
495 &snapshot,
496 &error,
497 config.failure_action,
498 );
499 if should_stop {
500 stopped_by_policy = true;
501 break 'heartbeat;
502 }
503 continue;
504 }
505 };
506 let sent_at = SystemTime::now();
507 let builder = sender.message(config.payload.clone()).options(options);
508 let send = async {
509 if reconnect.is_some() {
510 let receipt = builder.enqueue().await.map_err(crate::EnqueueError::into_error)?;
511 receipt.written_connection().await.map(Some)
512 } else {
513 builder.send().await.map(|_| None)
514 }
515 };
516 let send_result = tokio::select! {
517 _ = cancellation.cancelled() => break 'heartbeat,
518 _ = session_closed.cancelled() => break 'heartbeat,
519 result = send => result,
520 };
521 let written_connection = match send_result {
522 Ok(connection) => connection,
523 Err(error) => {
524 diagnostics.error(session_id, HeartbeatPhase::Send, &error);
525 missed_streak.count = 0;
526 if record_failure(&snapshot, &error, config.failure_action) {
527 stopped_by_policy = true;
528 break 'heartbeat;
529 }
530 continue;
531 }
532 };
533 snapshot_update(&snapshot, |current| {
534 increment(&mut current.sent);
535 });
536 let (Some(matcher), Some(receiver)) =
537 (config.matcher.as_ref(), receiver.as_mut())
538 else {
539 diagnostics.recovered(session_id);
540 continue;
541 };
542 let connection = match written_connection {
545 Some(connection) => Ok(Some(connection)),
546 None => runtime.current_connection_id(),
547 };
548 let connection_id = match connection {
549 Ok(Some(connection)) => connection,
550 other => {
551 let error = match other {
552 Err(error) => {
553 diagnostics.error(session_id, HeartbeatPhase::Connection, &error);
554 error
555 }
556 Ok(_) => {
557 if config.failure_action == ApplicationHeartbeatFailureAction::Stop {
560 diagnostics.last = Some(HeartbeatFailure {
561 phase: HeartbeatPhase::Connection,
562 kind: ErrorKind::NotConnected,
563 stage: Some(ErrorStage::Runtime),
564 });
565 diagnostics.failures = diagnostics.failures.saturating_add(1);
566 }
567 NetError::from(ErrorKind::NotConnected).with_stage(ErrorStage::Runtime)
568 }
569 };
570 missed_streak.count = 0;
571 if record_failure(&snapshot, &error, config.failure_action) {
572 stopped_by_policy = true;
573 break 'heartbeat;
574 }
575 continue;
576 }
577 };
578 snapshot_update(&snapshot, |current| {
579 current.state = ApplicationHeartbeatState::WaitingForResponse;
580 });
581 let timeout = match config.response_timeout {
582 Some(timeout) => timeout,
583 None => continue,
584 };
585 match wait_for_response(
586 receiver,
587 matcher,
588 timeout,
589 &cancellation,
590 &session_closed,
591 connection_id,
592 sent_at,
593 ).await {
594 ResponseWait::Matched => {
595 missed_streak.count = 0;
596 snapshot_update(&snapshot, |current| {
597 increment(&mut current.matched_responses);
598 current.state = ApplicationHeartbeatState::Running;
599 });
600 diagnostics.recovered(session_id);
601 }
602 ResponseWait::TimedOut => {
603 diagnostics.failed(session_id, HeartbeatFailure {
604 phase: HeartbeatPhase::ResponseWait, kind: ErrorKind::TimedOut, stage: Some(ErrorStage::Receive),
605 });
606 let consecutive_missed = missed_streak.record(written_connection);
607 snapshot_update(&snapshot, |current| {
608 increment(&mut current.missed_responses);
609 current.last_error = Some(ErrorKind::TimedOut);
610 increment(&mut current.failures);
611 current.state = ApplicationHeartbeatState::Running;
612 });
613 if consecutive_missed < config.max_missed_responses {
614 continue;
615 }
616 if let Some(client) = &reconnect {
617 missed_streak.count = 0;
618 let result = tokio::select! {
619 biased;
620 _ = cancellation.cancelled() => break 'heartbeat,
621 _ = session_closed.cancelled() => break 'heartbeat,
622 result = recover_connection(client, &runtime, connection_id, &cancellation) => result,
623 };
624 if let Err(error) = result {
625 if runtime.is_closed() || matches!(error.kind(), ErrorKind::Cancelled | ErrorKind::Closed | ErrorKind::EngineDropped) {
626 break 'heartbeat;
627 }
628 diagnostics.error(session_id, HeartbeatPhase::Connection, &error);
629 if record_failure(&snapshot, &error, config.failure_action) {
630 stopped_by_policy = true;
631 break 'heartbeat;
632 }
633 }
634 } else if config.failure_action == ApplicationHeartbeatFailureAction::Stop {
635 stopped_by_policy = true;
636 break 'heartbeat;
637 }
638 }
639 ResponseWait::Closed | ResponseWait::Cancelled => break 'heartbeat,
640 ResponseWait::Failed(failure) => {
641 diagnostics.failed(session_id, failure);
642 missed_streak.count = 0;
645 let should_stop = record_failure_kind(
646 &snapshot,
647 failure.kind,
648 config.failure_action,
649 );
650 if should_stop {
651 stopped_by_policy = true;
652 break 'heartbeat;
653 }
654 }
655 }
656 }
657 }
658 }
659 if let Some(receiver) = receiver.as_ref() {
660 receiver.unsubscribe();
661 }
662 snapshot_update(&snapshot, |current| {
663 current.state = ApplicationHeartbeatState::Stopped;
664 });
665 if stopped_by_policy {
666 diagnostics.stopped(session_id);
667 }
668}
669
670#[derive(Default)]
671struct MissedResponses {
672 connection: Option<ConnectionId>,
673 count: u32,
674}
675
676impl MissedResponses {
677 fn record(&mut self, connection: Option<ConnectionId>) -> u32 {
680 if self.connection != connection {
681 self.connection = connection;
682 self.count = 0;
683 }
684 self.count = self.count.saturating_add(1);
685 self.count
686 }
687}
688
689async fn recover_connection(
690 client: &Weak<WSClientInner>,
691 runtime: &Arc<SessionRuntime>,
692 connection: ConnectionId,
693 cancellation: &CancellationToken,
694) -> Result<()> {
695 client
696 .upgrade()
697 .ok_or_else(|| NetError::from(ErrorKind::Closed))?
698 .reconnect_if_current(runtime, connection, Some(cancellation.clone()))
699 .await?;
700 runtime
701 .lifecycle
702 .wait_connected_after(Some(connection))
703 .await?;
704 Ok(())
705}
706
707async fn wait_for_response(
708 receiver: &mut MessageReceiver,
709 matcher: &ApplicationHeartbeatMatcher,
710 timeout: Duration,
711 cancellation: &CancellationToken,
712 session_closed: &CancellationToken,
713 connection_id: ConnectionId,
714 sent_at: SystemTime,
715) -> ResponseWait {
716 let deadline = time::sleep(timeout);
717 tokio::pin!(deadline);
718 loop {
719 tokio::select! {
720 _ = cancellation.cancelled() => return ResponseWait::Cancelled,
721 _ = session_closed.cancelled() => return ResponseWait::Closed,
722 _ = &mut deadline => return ResponseWait::TimedOut,
723 result = receiver.recv() => {
724 match result {
725 Ok(Some(message)) => {
726 if connection_id != message.connection_id() {
727 continue;
728 }
729 if message.received_at().duration_since(sent_at).is_err() {
730 continue;
731 }
732 let matches = match catch_unwind(AssertUnwindSafe(|| matcher(&message))) {
733 Ok(matches) => matches,
734 Err(_) => return ResponseWait::Failed(HeartbeatFailure {
735 phase: HeartbeatPhase::Matcher, kind: ErrorKind::CallbackPanicked, stage: Some(ErrorStage::Receive),
736 }),
737 };
738 if matches {
739 return ResponseWait::Matched;
740 }
741 }
742 Ok(None) => return ResponseWait::Closed,
743 Err(ReceiveError::Lagged { .. }) => continue,
749 Err(ReceiveError::Failed(error)) => {
750 return ResponseWait::Failed(HeartbeatFailure {
751 phase: HeartbeatPhase::Receive, kind: error.kind(), stage: error.context().stage,
752 });
753 }
754 }
755 }
756 }
757 }
758}
759
760fn send_options(timeout: Duration) -> Result<SendOptions> {
761 let deadline = StdInstant::now().checked_add(timeout).ok_or_else(|| {
762 NetError::config(
763 "application_heartbeat.send_timeout",
764 "cannot be represented as a deadline",
765 )
766 })?;
767 Ok(SendOptions {
768 disconnected: DisconnectedPolicy::WaitForReconnect,
769 enqueue_timeout: Some(timeout),
770 write_timeout: timeout,
771 deadline: Some(deadline),
772 ..SendOptions::default()
773 })
774}
775
776fn record_failure(
777 snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
778 error: &NetError,
779 action: ApplicationHeartbeatFailureAction,
780) -> bool {
781 record_failure_kind(snapshot, error.kind(), action)
782}
783
784fn record_failure_kind(
785 snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
786 kind: ErrorKind,
787 action: ApplicationHeartbeatFailureAction,
788) -> bool {
789 snapshot_update(snapshot, |current| {
790 increment(&mut current.failures);
791 current.last_error = Some(kind);
792 current.state = ApplicationHeartbeatState::Running;
793 });
794 action == ApplicationHeartbeatFailureAction::Stop
795}
796
797fn increment(value: &mut u64) {
798 if let Some(next) = value.checked_add(1) {
799 *value = next;
800 }
801}
802
803fn snapshot_copy(
804 snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
805) -> ApplicationHeartbeatSnapshot {
806 *lock_snapshot(snapshot)
807}
808
809fn snapshot_update<F>(snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>, update: F)
810where
811 F: FnOnce(&mut ApplicationHeartbeatSnapshot),
812{
813 let mut current = lock_snapshot(snapshot);
814 update(&mut current);
815}
816
817fn lock_snapshot<'a>(
818 snapshot: &'a Arc<Mutex<ApplicationHeartbeatSnapshot>>,
819) -> MutexGuard<'a, ApplicationHeartbeatSnapshot> {
820 match snapshot.lock() {
821 Ok(guard) => guard,
822 Err(poisoned) => {
823 crate::log_e!(crate::common::log::log_def::LogType::WSC;
824 "ws.heartbeat.lock_snapshot", "arg_snapshot_present|error", true, "lock_poisoned_recovered");
825 poisoned.into_inner()
826 }
827 }
828}
829
830#[cfg(test)]
831#[path = "application_heartbeat_tests.rs"]
832mod tests;