1use super::{
15 DisconnectedPolicy, IncomingMessage, Message, MessageReceiver, ReceiveOptions, ReceiveOverflow,
16 SendOptions, Sender,
17};
18use crate::error::{ErrorKind, ErrorStage, ReceiveError};
19use crate::module::ws_client::session_runtime::SessionRuntime;
20use crate::{NetError, Result};
21use std::panic::{catch_unwind, AssertUnwindSafe};
22use std::sync::{Arc, Mutex, MutexGuard};
23use std::time::{Duration, Instant as StdInstant, SystemTime};
24use tokio::task::JoinHandle;
25use tokio::time::{self, Instant};
26use tokio_util::sync::CancellationToken;
27
28pub type ApplicationHeartbeatMatcher =
36 Arc<dyn Fn(&IncomingMessage) -> bool + Send + Sync + 'static>;
37
38#[derive(Clone, Copy, Debug, Eq, PartialEq)]
40pub enum ApplicationHeartbeatFailureAction {
41 Continue,
43 Stop,
45}
46
47#[derive(Clone)]
49pub struct ApplicationHeartbeatConfig {
50 pub interval: Duration,
52 pub payload: Message,
54 pub send_timeout: Duration,
56 pub response_timeout: Option<Duration>,
59 pub max_missed_responses: u32,
61 pub matcher: Option<ApplicationHeartbeatMatcher>,
63 pub receive_options: ReceiveOptions,
68 pub failure_action: ApplicationHeartbeatFailureAction,
70}
71
72impl ApplicationHeartbeatConfig {
73 pub fn new(interval: Duration, payload: impl Into<Message>) -> Self {
75 Self {
76 interval,
77 payload: payload.into(),
78 send_timeout: Duration::from_secs(10),
79 response_timeout: None,
80 max_missed_responses: 1,
81 matcher: None,
82 receive_options: ReceiveOptions {
83 overflow: ReceiveOverflow::DropOldest,
84 ..ReceiveOptions::default()
85 },
86 failure_action: ApplicationHeartbeatFailureAction::Continue,
87 }
88 }
89
90 pub fn with_send_timeout(mut self, timeout: Duration) -> Self {
92 self.send_timeout = timeout;
93 self
94 }
95
96 pub fn with_response_matcher(
98 mut self,
99 timeout: Duration,
100 matcher: impl Fn(&IncomingMessage) -> bool + Send + Sync + 'static,
101 ) -> Self {
102 self.response_timeout = Some(timeout);
103 self.matcher = Some(Arc::new(matcher));
104 self
105 }
106
107 pub fn with_max_missed_responses(mut self, max_missed_responses: u32) -> Self {
109 self.max_missed_responses = max_missed_responses;
110 self
111 }
112
113 pub fn with_receive_options(mut self, receive_options: ReceiveOptions) -> Self {
115 self.receive_options = receive_options;
116 self
117 }
118
119 pub fn with_failure_action(
121 mut self,
122 failure_action: ApplicationHeartbeatFailureAction,
123 ) -> Self {
124 self.failure_action = failure_action;
125 self
126 }
127
128 pub fn validate(&self) -> Result<()> {
130 if self.interval.is_zero() {
131 return Err(NetError::config(
132 "application_heartbeat.interval",
133 "must be greater than zero",
134 ));
135 }
136 if Instant::now().checked_add(self.interval).is_none() {
137 return Err(NetError::config(
138 "application_heartbeat.interval",
139 "cannot be represented as a timer deadline",
140 ));
141 }
142 if self.send_timeout.is_zero() {
143 return Err(NetError::config(
144 "application_heartbeat.send_timeout",
145 "must be greater than zero",
146 ));
147 }
148 if StdInstant::now().checked_add(self.send_timeout).is_none() {
149 return Err(NetError::config(
150 "application_heartbeat.send_timeout",
151 "cannot be represented as a deadline",
152 ));
153 }
154 if self.max_missed_responses == 0 {
155 return Err(NetError::config(
156 "application_heartbeat.max_missed_responses",
157 "must be greater than zero",
158 ));
159 }
160 if self.matcher.is_some() && self.receive_options.include_control_frames {
161 return Err(NetError::config(
162 "application_heartbeat.receive_options.include_control_frames",
163 "must be false so control Pong frames cannot acknowledge application heartbeats",
164 ));
165 }
166 if self.matcher.is_some() && self.response_timeout.is_none() {
167 return Err(NetError::config(
168 "application_heartbeat.response_timeout",
169 "is required when matcher is set",
170 ));
171 }
172 if let Some(timeout) = self.response_timeout {
173 if StdInstant::now().checked_add(timeout).is_none() {
174 return Err(NetError::config(
175 "application_heartbeat.response_timeout",
176 "must fit an absolute deadline",
177 ));
178 }
179 if timeout.is_zero() {
180 return Err(NetError::config(
181 "application_heartbeat.response_timeout",
182 "must be greater than zero",
183 ));
184 }
185 if Instant::now().checked_add(timeout).is_none() {
186 return Err(NetError::config(
187 "application_heartbeat.response_timeout",
188 "cannot be represented as a deadline",
189 ));
190 }
191 if self.matcher.is_none() {
192 return Err(NetError::config(
193 "application_heartbeat.matcher",
194 "is required when response_timeout is set",
195 ));
196 }
197 }
198 if self.matcher.is_some() {
199 self.receive_options.validate()?;
200 }
201 Ok(())
202 }
203}
204
205impl std::fmt::Debug for ApplicationHeartbeatConfig {
206 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
207 formatter
208 .debug_struct("ApplicationHeartbeatConfig")
209 .field("interval", &self.interval)
210 .field("payload_bytes", &self.payload.len())
211 .field("send_timeout", &self.send_timeout)
212 .field("response_timeout", &self.response_timeout)
213 .field("max_missed_responses", &self.max_missed_responses)
214 .field("has_matcher", &self.matcher.is_some())
215 .field("receive_options", &self.receive_options)
216 .field("failure_action", &self.failure_action)
217 .finish()
218 }
219}
220
221#[derive(Clone, Copy, Debug, Eq, PartialEq)]
223pub enum ApplicationHeartbeatState {
224 Running,
226 WaitingForResponse,
228 Stopped,
230}
231
232#[derive(Clone, Copy, Debug, Eq, PartialEq)]
234pub struct ApplicationHeartbeatSnapshot {
235 pub state: ApplicationHeartbeatState,
237 pub sent: u64,
239 pub matched_responses: u64,
241 pub missed_responses: u64,
243 pub failures: u64,
245 pub last_error: Option<ErrorKind>,
247}
248
249impl Default for ApplicationHeartbeatSnapshot {
250 fn default() -> Self {
251 Self {
252 state: ApplicationHeartbeatState::Running,
253 sent: 0,
254 matched_responses: 0,
255 missed_responses: 0,
256 failures: 0,
257 last_error: None,
258 }
259 }
260}
261
262pub struct ApplicationHeartbeatHandle {
267 cancellation: CancellationToken,
268 snapshot: Arc<Mutex<ApplicationHeartbeatSnapshot>>,
269 task: Option<JoinHandle<()>>,
270}
271
272impl ApplicationHeartbeatHandle {
273 pub(crate) fn new(
274 sender: Sender,
275 receiver: Option<MessageReceiver>,
276 config: ApplicationHeartbeatConfig,
277 ) -> Result<Self> {
278 config.validate()?;
279 let cancellation = CancellationToken::new();
280 let snapshot = Arc::new(Mutex::new(ApplicationHeartbeatSnapshot::default()));
281 let runtime = sender.runtime.clone();
282 let task = runtime.clone().spawn(run_heartbeat(
283 sender,
284 receiver,
285 config,
286 cancellation.clone(),
287 snapshot.clone(),
288 runtime,
289 ))?;
290 Ok(Self {
291 cancellation,
292 snapshot,
293 task: Some(task),
294 })
295 }
296
297 pub fn stop(&self) {
299 self.cancellation.cancel();
300 }
301
302 pub fn snapshot(&self) -> ApplicationHeartbeatSnapshot {
304 snapshot_copy(&self.snapshot)
305 }
306
307 pub async fn stopped(mut self) -> Result<ApplicationHeartbeatSnapshot> {
309 let task = self
310 .task
311 .take()
312 .ok_or_else(|| NetError::from(ErrorKind::Internal).with_stage(ErrorStage::Runtime))?;
313 task.await.map_err(|error| {
314 NetError::with_source(ErrorKind::EngineDropped, error).with_stage(ErrorStage::Runtime)
315 })?;
316 Ok(snapshot_copy(&self.snapshot))
317 }
318}
319
320impl Drop for ApplicationHeartbeatHandle {
321 fn drop(&mut self) {
322 self.cancellation.cancel();
323 if let Some(task) = self.task.take() {
324 task.abort();
325 }
326 }
327}
328
329#[derive(Clone, Copy, Debug, Eq, PartialEq)]
330enum ResponseWait {
331 Matched,
332 TimedOut,
333 Closed,
334 Cancelled,
335 Failed(HeartbeatFailure),
336}
337
338#[derive(Clone, Copy, Debug, Eq, PartialEq)]
339enum HeartbeatPhase {
340 SendOptions,
341 Send,
342 Connection,
343 ResponseWait,
344 Matcher,
345 Receive,
346}
347
348impl HeartbeatPhase {
349 fn name(self) -> &'static str {
350 match self {
351 Self::SendOptions => "send_options",
352 Self::Send => "send",
353 Self::Connection => "connection",
354 Self::ResponseWait => "response_wait",
355 Self::Matcher => "matcher",
356 Self::Receive => "receive",
357 }
358 }
359}
360
361#[derive(Clone, Copy, Debug, Eq, PartialEq)]
362struct HeartbeatFailure {
363 phase: HeartbeatPhase,
364 kind: ErrorKind,
365 stage: Option<ErrorStage>,
366}
367
368const HEARTBEAT_STAGE_SLOTS: usize = ErrorStage::Runtime as usize + 2;
369
370#[derive(Default)]
371struct HeartbeatDiagnostics {
372 client_id: u64,
373 last: Option<HeartbeatFailure>,
374 failures: u64,
375 reported: [[u64; HEARTBEAT_STAGE_SLOTS]; 6],
378}
379
380impl HeartbeatDiagnostics {
381 fn should_report(&mut self, failure: HeartbeatFailure, enabled: bool) -> bool {
382 if matches!(failure.kind, ErrorKind::Cancelled | ErrorKind::Closed) {
383 self.last = None;
384 self.failures = 0;
385 self.reported = [[0; HEARTBEAT_STAGE_SLOTS]; 6];
386 return false;
387 }
388 self.last = Some(failure);
389 self.failures = self.failures.saturating_add(1);
390 if !enabled {
391 self.reported = [[0; HEARTBEAT_STAGE_SLOTS]; 6];
394 return false;
395 }
396 let bit = 1_u64
399 .checked_shl(failure.kind as u32)
400 .unwrap_or(1_u64 << 63);
401 let stage = failure
402 .stage
403 .map_or(0, |stage| stage as usize + 1)
404 .min(HEARTBEAT_STAGE_SLOTS - 1);
405 let reported = &mut self.reported[failure.phase as usize][stage];
406 let first = *reported & bit == 0;
407 *reported |= bit;
408 first
409 }
410
411 fn failed(&mut self, session_id: u64, failure: HeartbeatFailure) {
412 if self.should_report(
413 failure,
414 crate::OpenNetLogger::is_enabled_for(crate::LogType::WSC, crate::LogLevel::Error),
415 ) {
416 crate::log_e!(crate::LogType::WSC; "ws.heartbeat.failed",
417 "client_id|session_id|phase|kind|stage|failures", self.client_id, session_id, failure.phase.name(),
418 format!("{:?}", failure.kind), failure.stage.map(|stage| format!("{stage:?}")), self.failures);
419 }
420 }
421
422 fn error(&mut self, session_id: u64, phase: HeartbeatPhase, error: &NetError) {
423 self.failed(
424 session_id,
425 HeartbeatFailure {
426 phase,
427 kind: error.kind(),
428 stage: error.context().stage,
429 },
430 );
431 }
432
433 fn recovered(&mut self, session_id: u64) {
434 if let Some(failure) = self.last.take() {
435 crate::log_s!(crate::LogType::WSC, crate::LogLevel::Key; "ws.heartbeat.recovered",
436 "client_id|session_id|phase|failures", self.client_id, session_id, failure.phase.name(), self.failures);
437 }
438 self.failures = 0;
439 self.reported = [[0; HEARTBEAT_STAGE_SLOTS]; 6];
440 }
441
442 fn stopped(&self, session_id: u64) {
443 if let Some(failure) = self.last {
444 crate::log_s!(crate::LogType::WSC, crate::LogLevel::Key; "ws.heartbeat.stopped",
445 "client_id|session_id|phase|kind|failures", self.client_id, session_id, failure.phase.name(), format!("{:?}", failure.kind), self.failures);
446 }
447 }
448}
449
450async fn run_heartbeat(
451 sender: Sender,
452 mut receiver: Option<MessageReceiver>,
453 config: ApplicationHeartbeatConfig,
454 cancellation: CancellationToken,
455 snapshot: Arc<Mutex<ApplicationHeartbeatSnapshot>>,
456 runtime: Arc<SessionRuntime>,
457) {
458 let now = Instant::now();
459 let first_tick = match now.checked_add(config.interval) {
460 Some(deadline) => deadline,
461 None => now,
462 };
463 let mut ticker = time::interval_at(first_tick, config.interval);
464 ticker.set_missed_tick_behavior(time::MissedTickBehavior::Skip);
465 let mut consecutive_missed = 0_u32;
466 let mut stopped_by_policy = false;
467 let session_closed = runtime.closed_token();
468 let session_id = runtime.id.as_u64();
469 let mut diagnostics = HeartbeatDiagnostics {
470 client_id: runtime.client_id.as_u64(),
471 ..Default::default()
472 };
473
474 'heartbeat: loop {
475 tokio::select! {
476 _ = cancellation.cancelled() => break 'heartbeat,
477 _ = session_closed.cancelled() => break 'heartbeat,
478 _ = ticker.tick() => {
479 snapshot_update(&snapshot, |current| {
480 current.state = ApplicationHeartbeatState::Running;
481 });
482 let options = match send_options(config.send_timeout) {
483 Ok(options) => options,
484 Err(error) => {
485 diagnostics.error(session_id, HeartbeatPhase::SendOptions, &error);
486 consecutive_missed = 0;
490 let should_stop = record_failure(
491 &snapshot,
492 &error,
493 config.failure_action,
494 );
495 if should_stop {
496 stopped_by_policy = true;
497 break 'heartbeat;
498 }
499 continue;
500 }
501 };
502 let sent_at = SystemTime::now();
503 let send = sender
504 .message(config.payload.clone())
505 .options(options)
506 .send();
507 let send_result = tokio::select! {
508 _ = cancellation.cancelled() => break 'heartbeat,
509 _ = session_closed.cancelled() => break 'heartbeat,
510 result = send => result,
511 };
512 if let Err(error) = send_result {
513 diagnostics.error(session_id, HeartbeatPhase::Send, &error);
514 consecutive_missed = 0;
517 let should_stop = record_failure(
518 &snapshot,
519 &error,
520 config.failure_action,
521 );
522 if should_stop {
523 stopped_by_policy = true;
524 break 'heartbeat;
525 }
526 continue;
527 }
528 snapshot_update(&snapshot, |current| {
529 increment(&mut current.sent);
530 });
531 let (Some(matcher), Some(receiver)) =
532 (config.matcher.as_ref(), receiver.as_mut())
533 else {
534 diagnostics.recovered(session_id);
535 continue;
536 };
537 let connection_id = match runtime.current_connection_id() {
538 Ok(Some(connection_id)) => connection_id,
539 Ok(None) => {
540 consecutive_missed = 0;
545 let error = NetError::from(ErrorKind::NotConnected)
546 .with_stage(ErrorStage::Runtime);
547 let should_stop = record_failure(
548 &snapshot,
549 &error,
550 config.failure_action,
551 );
552 if should_stop {
553 diagnostics.last = Some(HeartbeatFailure {
556 phase: HeartbeatPhase::Connection,
557 kind: ErrorKind::NotConnected,
558 stage: Some(ErrorStage::Runtime),
559 });
560 diagnostics.failures = diagnostics.failures.saturating_add(1);
561 stopped_by_policy = true;
562 break 'heartbeat;
563 }
564 continue;
565 }
566 Err(error) => {
567 diagnostics.error(session_id, HeartbeatPhase::Connection, &error);
568 consecutive_missed = 0;
572 let should_stop = record_failure(
573 &snapshot,
574 &error,
575 config.failure_action,
576 );
577 if should_stop {
578 stopped_by_policy = true;
579 break 'heartbeat;
580 }
581 continue;
582 }
583 };
584 snapshot_update(&snapshot, |current| {
585 current.state = ApplicationHeartbeatState::WaitingForResponse;
586 });
587 let timeout = match config.response_timeout {
588 Some(timeout) => timeout,
589 None => continue,
590 };
591 match wait_for_response(
592 receiver,
593 matcher,
594 timeout,
595 &cancellation,
596 &session_closed,
597 Some(connection_id),
598 sent_at,
599 ).await {
600 ResponseWait::Matched => {
601 consecutive_missed = 0;
602 snapshot_update(&snapshot, |current| {
603 increment(&mut current.matched_responses);
604 current.state = ApplicationHeartbeatState::Running;
605 });
606 diagnostics.recovered(session_id);
607 }
608 ResponseWait::TimedOut => {
609 diagnostics.failed(session_id, HeartbeatFailure {
610 phase: HeartbeatPhase::ResponseWait, kind: ErrorKind::TimedOut, stage: Some(ErrorStage::Receive),
611 });
612 consecutive_missed = consecutive_missed.saturating_add(1);
613 snapshot_update(&snapshot, |current| {
614 increment(&mut current.missed_responses);
615 current.last_error = Some(ErrorKind::TimedOut);
616 increment(&mut current.failures);
617 current.state = ApplicationHeartbeatState::Running;
618 });
619 if config.failure_action == ApplicationHeartbeatFailureAction::Stop
620 && consecutive_missed >= config.max_missed_responses
621 {
622 stopped_by_policy = true;
623 break 'heartbeat;
624 }
625 }
626 ResponseWait::Closed | ResponseWait::Cancelled => break 'heartbeat,
627 ResponseWait::Failed(failure) => {
628 diagnostics.failed(session_id, failure);
629 consecutive_missed = 0;
632 let should_stop = record_failure_kind(
633 &snapshot,
634 failure.kind,
635 config.failure_action,
636 );
637 if should_stop {
638 stopped_by_policy = true;
639 break 'heartbeat;
640 }
641 }
642 }
643 }
644 }
645 }
646 if let Some(receiver) = receiver.as_ref() {
647 receiver.unsubscribe();
648 }
649 snapshot_update(&snapshot, |current| {
650 current.state = ApplicationHeartbeatState::Stopped;
651 });
652 if stopped_by_policy {
653 diagnostics.stopped(session_id);
654 }
655}
656
657async fn wait_for_response(
658 receiver: &mut MessageReceiver,
659 matcher: &ApplicationHeartbeatMatcher,
660 timeout: Duration,
661 cancellation: &CancellationToken,
662 session_closed: &CancellationToken,
663 connection_id: Option<crate::ws::ConnectionId>,
664 sent_at: SystemTime,
665) -> ResponseWait {
666 let deadline = time::sleep(timeout);
667 tokio::pin!(deadline);
668 loop {
669 tokio::select! {
670 _ = cancellation.cancelled() => return ResponseWait::Cancelled,
671 _ = session_closed.cancelled() => return ResponseWait::Closed,
672 _ = &mut deadline => return ResponseWait::TimedOut,
673 result = receiver.recv() => {
674 match result {
675 Ok(Some(message)) => {
676 if connection_id != Some(message.connection_id()) {
677 continue;
678 }
679 if message.received_at().duration_since(sent_at).is_err() {
680 continue;
681 }
682 let matches = match catch_unwind(AssertUnwindSafe(|| matcher(&message))) {
683 Ok(matches) => matches,
684 Err(_) => return ResponseWait::Failed(HeartbeatFailure {
685 phase: HeartbeatPhase::Matcher, kind: ErrorKind::CallbackPanicked, stage: Some(ErrorStage::Receive),
686 }),
687 };
688 if matches {
689 return ResponseWait::Matched;
690 }
691 }
692 Ok(None) => return ResponseWait::Closed,
693 Err(ReceiveError::Lagged { .. }) => continue,
699 Err(ReceiveError::Failed(error)) => {
700 return ResponseWait::Failed(HeartbeatFailure {
701 phase: HeartbeatPhase::Receive, kind: error.kind(), stage: error.context().stage,
702 });
703 }
704 }
705 }
706 }
707 }
708}
709
710fn send_options(timeout: Duration) -> Result<SendOptions> {
711 let deadline = StdInstant::now().checked_add(timeout).ok_or_else(|| {
712 NetError::config(
713 "application_heartbeat.send_timeout",
714 "cannot be represented as a deadline",
715 )
716 })?;
717 Ok(SendOptions {
718 disconnected: DisconnectedPolicy::WaitForReconnect,
719 enqueue_timeout: Some(timeout),
720 write_timeout: timeout,
721 deadline: Some(deadline),
722 ..SendOptions::default()
723 })
724}
725
726fn record_failure(
727 snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
728 error: &NetError,
729 action: ApplicationHeartbeatFailureAction,
730) -> bool {
731 record_failure_kind(snapshot, error.kind(), action)
732}
733
734fn record_failure_kind(
735 snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
736 kind: ErrorKind,
737 action: ApplicationHeartbeatFailureAction,
738) -> bool {
739 snapshot_update(snapshot, |current| {
740 increment(&mut current.failures);
741 current.last_error = Some(kind);
742 current.state = ApplicationHeartbeatState::Running;
743 });
744 action == ApplicationHeartbeatFailureAction::Stop
745}
746
747fn increment(value: &mut u64) {
748 if let Some(next) = value.checked_add(1) {
749 *value = next;
750 }
751}
752
753fn snapshot_copy(
754 snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
755) -> ApplicationHeartbeatSnapshot {
756 *lock_snapshot(snapshot)
757}
758
759fn snapshot_update<F>(snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>, update: F)
760where
761 F: FnOnce(&mut ApplicationHeartbeatSnapshot),
762{
763 let mut current = lock_snapshot(snapshot);
764 update(&mut current);
765}
766
767fn lock_snapshot<'a>(
768 snapshot: &'a Arc<Mutex<ApplicationHeartbeatSnapshot>>,
769) -> MutexGuard<'a, ApplicationHeartbeatSnapshot> {
770 match snapshot.lock() {
771 Ok(guard) => guard,
772 Err(poisoned) => {
773 crate::log_e!(crate::common::log::log_def::LogType::WSC;
774 "ws.heartbeat.lock_snapshot", "arg_snapshot_present|error", true, "lock_poisoned_recovered");
775 poisoned.into_inner()
776 }
777 }
778}
779
780#[cfg(test)]
781#[path = "application_heartbeat_tests.rs"]
782mod tests;