1use super::{
8 DisconnectedPolicy, IncomingMessage, Message, MessageReceiver, ReceiveOptions, ReceiveOverflow,
9 SendOptions, Sender,
10};
11use crate::error::{ErrorKind, ErrorStage, ReceiveError};
12use crate::module::ws_client::session_runtime::SessionRuntime;
13use crate::{NetError, Result};
14use std::panic::{catch_unwind, AssertUnwindSafe};
15use std::sync::{Arc, Mutex, MutexGuard};
16use std::time::{Duration, Instant as StdInstant, SystemTime};
17use tokio::task::JoinHandle;
18use tokio::time::{self, Instant};
19use tokio_util::sync::CancellationToken;
20
21pub type ApplicationHeartbeatMatcher =
29 Arc<dyn Fn(&IncomingMessage) -> bool + Send + Sync + 'static>;
30
31#[derive(Clone, Copy, Debug, Eq, PartialEq)]
33pub enum ApplicationHeartbeatFailureAction {
34 Continue,
36 Stop,
38}
39
40#[derive(Clone)]
42pub struct ApplicationHeartbeatConfig {
43 pub interval: Duration,
45 pub payload: Message,
47 pub send_timeout: Duration,
49 pub response_timeout: Option<Duration>,
52 pub max_missed_responses: u32,
54 pub matcher: Option<ApplicationHeartbeatMatcher>,
56 pub receive_options: ReceiveOptions,
61 pub failure_action: ApplicationHeartbeatFailureAction,
63}
64
65impl ApplicationHeartbeatConfig {
66 pub fn new(interval: Duration, payload: impl Into<Message>) -> Self {
68 Self {
69 interval,
70 payload: payload.into(),
71 send_timeout: Duration::from_secs(10),
72 response_timeout: None,
73 max_missed_responses: 1,
74 matcher: None,
75 receive_options: ReceiveOptions {
76 overflow: ReceiveOverflow::DropOldest,
77 ..ReceiveOptions::default()
78 },
79 failure_action: ApplicationHeartbeatFailureAction::Continue,
80 }
81 }
82
83 pub fn with_send_timeout(mut self, timeout: Duration) -> Self {
85 self.send_timeout = timeout;
86 self
87 }
88
89 pub fn with_response_matcher(
91 mut self,
92 timeout: Duration,
93 matcher: impl Fn(&IncomingMessage) -> bool + Send + Sync + 'static,
94 ) -> Self {
95 self.response_timeout = Some(timeout);
96 self.matcher = Some(Arc::new(matcher));
97 self
98 }
99
100 pub fn with_max_missed_responses(mut self, max_missed_responses: u32) -> Self {
102 self.max_missed_responses = max_missed_responses;
103 self
104 }
105
106 pub fn with_receive_options(mut self, receive_options: ReceiveOptions) -> Self {
108 self.receive_options = receive_options;
109 self
110 }
111
112 pub fn with_failure_action(
114 mut self,
115 failure_action: ApplicationHeartbeatFailureAction,
116 ) -> Self {
117 self.failure_action = failure_action;
118 self
119 }
120
121 pub fn validate(&self) -> Result<()> {
123 if self.interval.is_zero() {
124 return Err(NetError::config(
125 "application_heartbeat.interval",
126 "must be greater than zero",
127 ));
128 }
129 if Instant::now().checked_add(self.interval).is_none() {
130 return Err(NetError::config(
131 "application_heartbeat.interval",
132 "cannot be represented as a timer deadline",
133 ));
134 }
135 if self.send_timeout.is_zero() {
136 return Err(NetError::config(
137 "application_heartbeat.send_timeout",
138 "must be greater than zero",
139 ));
140 }
141 if StdInstant::now().checked_add(self.send_timeout).is_none() {
142 return Err(NetError::config(
143 "application_heartbeat.send_timeout",
144 "cannot be represented as a deadline",
145 ));
146 }
147 if self.max_missed_responses == 0 {
148 return Err(NetError::config(
149 "application_heartbeat.max_missed_responses",
150 "must be greater than zero",
151 ));
152 }
153 if self.matcher.is_some() && self.receive_options.include_control_frames {
154 return Err(NetError::config(
155 "application_heartbeat.receive_options.include_control_frames",
156 "must be false so control Pong frames cannot acknowledge application heartbeats",
157 ));
158 }
159 if self.matcher.is_some() && self.response_timeout.is_none() {
160 return Err(NetError::config(
161 "application_heartbeat.response_timeout",
162 "is required when matcher is set",
163 ));
164 }
165 if let Some(timeout) = self.response_timeout {
166 if StdInstant::now().checked_add(timeout).is_none() {
167 return Err(NetError::config(
168 "application_heartbeat.response_timeout",
169 "must fit an absolute deadline",
170 ));
171 }
172 if timeout.is_zero() {
173 return Err(NetError::config(
174 "application_heartbeat.response_timeout",
175 "must be greater than zero",
176 ));
177 }
178 if Instant::now().checked_add(timeout).is_none() {
179 return Err(NetError::config(
180 "application_heartbeat.response_timeout",
181 "cannot be represented as a deadline",
182 ));
183 }
184 if self.matcher.is_none() {
185 return Err(NetError::config(
186 "application_heartbeat.matcher",
187 "is required when response_timeout is set",
188 ));
189 }
190 }
191 if self.matcher.is_some() {
192 self.receive_options.validate()?;
193 }
194 Ok(())
195 }
196}
197
198impl std::fmt::Debug for ApplicationHeartbeatConfig {
199 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
200 formatter
201 .debug_struct("ApplicationHeartbeatConfig")
202 .field("interval", &self.interval)
203 .field("payload_bytes", &self.payload.len())
204 .field("send_timeout", &self.send_timeout)
205 .field("response_timeout", &self.response_timeout)
206 .field("max_missed_responses", &self.max_missed_responses)
207 .field("has_matcher", &self.matcher.is_some())
208 .field("receive_options", &self.receive_options)
209 .field("failure_action", &self.failure_action)
210 .finish()
211 }
212}
213
214#[derive(Clone, Copy, Debug, Eq, PartialEq)]
216pub enum ApplicationHeartbeatState {
217 Running,
219 WaitingForResponse,
221 Stopped,
223}
224
225#[derive(Clone, Copy, Debug, Eq, PartialEq)]
227pub struct ApplicationHeartbeatSnapshot {
228 pub state: ApplicationHeartbeatState,
230 pub sent: u64,
232 pub matched_responses: u64,
234 pub missed_responses: u64,
236 pub failures: u64,
238 pub last_error: Option<ErrorKind>,
240}
241
242impl Default for ApplicationHeartbeatSnapshot {
243 fn default() -> Self {
244 Self {
245 state: ApplicationHeartbeatState::Running,
246 sent: 0,
247 matched_responses: 0,
248 missed_responses: 0,
249 failures: 0,
250 last_error: None,
251 }
252 }
253}
254
255pub struct ApplicationHeartbeatHandle {
260 cancellation: CancellationToken,
261 snapshot: Arc<Mutex<ApplicationHeartbeatSnapshot>>,
262 task: Option<JoinHandle<()>>,
263}
264
265impl ApplicationHeartbeatHandle {
266 pub(crate) fn new(
267 sender: Sender,
268 receiver: Option<MessageReceiver>,
269 config: ApplicationHeartbeatConfig,
270 ) -> Result<Self> {
271 config.validate()?;
272 let cancellation = CancellationToken::new();
273 let snapshot = Arc::new(Mutex::new(ApplicationHeartbeatSnapshot::default()));
274 let runtime = sender.runtime.clone();
275 let task = runtime.clone().spawn(run_heartbeat(
276 sender,
277 receiver,
278 config,
279 cancellation.clone(),
280 snapshot.clone(),
281 runtime,
282 ))?;
283 Ok(Self {
284 cancellation,
285 snapshot,
286 task: Some(task),
287 })
288 }
289
290 pub fn stop(&self) {
292 self.cancellation.cancel();
293 }
294
295 pub fn snapshot(&self) -> ApplicationHeartbeatSnapshot {
297 snapshot_copy(&self.snapshot)
298 }
299
300 pub async fn stopped(mut self) -> Result<ApplicationHeartbeatSnapshot> {
302 let task = self
303 .task
304 .take()
305 .ok_or_else(|| NetError::from(ErrorKind::Internal).with_stage(ErrorStage::Runtime))?;
306 task.await.map_err(|error| {
307 NetError::with_source(ErrorKind::EngineDropped, error).with_stage(ErrorStage::Runtime)
308 })?;
309 Ok(snapshot_copy(&self.snapshot))
310 }
311}
312
313impl Drop for ApplicationHeartbeatHandle {
314 fn drop(&mut self) {
315 self.cancellation.cancel();
316 if let Some(task) = self.task.take() {
317 task.abort();
318 }
319 }
320}
321
322#[derive(Clone, Copy, Debug, Eq, PartialEq)]
323enum ResponseWait {
324 Matched,
325 TimedOut,
326 Closed,
327 Cancelled,
328 Failed(HeartbeatFailure),
329}
330
331#[derive(Clone, Copy, Debug, Eq, PartialEq)]
332enum HeartbeatPhase {
333 SendOptions,
334 Send,
335 Connection,
336 ResponseWait,
337 Matcher,
338 Receive,
339}
340
341impl HeartbeatPhase {
342 fn name(self) -> &'static str {
343 match self {
344 Self::SendOptions => "send_options",
345 Self::Send => "send",
346 Self::Connection => "connection",
347 Self::ResponseWait => "response_wait",
348 Self::Matcher => "matcher",
349 Self::Receive => "receive",
350 }
351 }
352}
353
354#[derive(Clone, Copy, Debug, Eq, PartialEq)]
355struct HeartbeatFailure {
356 phase: HeartbeatPhase,
357 kind: ErrorKind,
358 stage: Option<ErrorStage>,
359}
360
361const HEARTBEAT_STAGE_SLOTS: usize = ErrorStage::Runtime as usize + 2;
362
363#[derive(Default)]
364struct HeartbeatDiagnostics {
365 client_id: u64,
366 last: Option<HeartbeatFailure>,
367 failures: u64,
368 reported: [[u64; HEARTBEAT_STAGE_SLOTS]; 6],
371}
372
373impl HeartbeatDiagnostics {
374 fn should_report(&mut self, failure: HeartbeatFailure, enabled: bool) -> bool {
375 if matches!(failure.kind, ErrorKind::Cancelled | ErrorKind::Closed) {
376 self.last = None;
377 self.failures = 0;
378 self.reported = [[0; HEARTBEAT_STAGE_SLOTS]; 6];
379 return false;
380 }
381 self.last = Some(failure);
382 self.failures = self.failures.saturating_add(1);
383 if !enabled {
384 self.reported = [[0; HEARTBEAT_STAGE_SLOTS]; 6];
387 return false;
388 }
389 let bit = 1_u64
392 .checked_shl(failure.kind as u32)
393 .unwrap_or(1_u64 << 63);
394 let stage = failure
395 .stage
396 .map_or(0, |stage| stage as usize + 1)
397 .min(HEARTBEAT_STAGE_SLOTS - 1);
398 let reported = &mut self.reported[failure.phase as usize][stage];
399 let first = *reported & bit == 0;
400 *reported |= bit;
401 first
402 }
403
404 fn failed(&mut self, session_id: u64, failure: HeartbeatFailure) {
405 if self.should_report(
406 failure,
407 crate::OpenNetLogger::is_enabled_for(crate::LogType::WSC, crate::LogLevel::Error),
408 ) {
409 crate::log_e!(crate::LogType::WSC; "ws.heartbeat.failed",
410 "client_id|session_id|phase|kind|stage|failures", self.client_id, session_id, failure.phase.name(),
411 format!("{:?}", failure.kind), failure.stage.map(|stage| format!("{stage:?}")), self.failures);
412 }
413 }
414
415 fn error(&mut self, session_id: u64, phase: HeartbeatPhase, error: &NetError) {
416 self.failed(
417 session_id,
418 HeartbeatFailure {
419 phase,
420 kind: error.kind(),
421 stage: error.context().stage,
422 },
423 );
424 }
425
426 fn recovered(&mut self, session_id: u64) {
427 if let Some(failure) = self.last.take() {
428 crate::log_s!(crate::LogType::WSC, crate::LogLevel::Key; "ws.heartbeat.recovered",
429 "client_id|session_id|phase|failures", self.client_id, session_id, failure.phase.name(), self.failures);
430 }
431 self.failures = 0;
432 self.reported = [[0; HEARTBEAT_STAGE_SLOTS]; 6];
433 }
434
435 fn stopped(&self, session_id: u64) {
436 if let Some(failure) = self.last {
437 crate::log_s!(crate::LogType::WSC, crate::LogLevel::Key; "ws.heartbeat.stopped",
438 "client_id|session_id|phase|kind|failures", self.client_id, session_id, failure.phase.name(), format!("{:?}", failure.kind), self.failures);
439 }
440 }
441}
442
443async fn run_heartbeat(
444 sender: Sender,
445 mut receiver: Option<MessageReceiver>,
446 config: ApplicationHeartbeatConfig,
447 cancellation: CancellationToken,
448 snapshot: Arc<Mutex<ApplicationHeartbeatSnapshot>>,
449 runtime: Arc<SessionRuntime>,
450) {
451 let now = Instant::now();
452 let first_tick = match now.checked_add(config.interval) {
453 Some(deadline) => deadline,
454 None => now,
455 };
456 let mut ticker = time::interval_at(first_tick, config.interval);
457 ticker.set_missed_tick_behavior(time::MissedTickBehavior::Skip);
458 let mut consecutive_missed = 0_u32;
459 let mut stopped_by_policy = false;
460 let session_closed = runtime.closed_token();
461 let session_id = runtime.id.as_u64();
462 let mut diagnostics = HeartbeatDiagnostics {
463 client_id: runtime.client_id.as_u64(),
464 ..Default::default()
465 };
466
467 'heartbeat: loop {
468 tokio::select! {
469 _ = cancellation.cancelled() => break 'heartbeat,
470 _ = session_closed.cancelled() => break 'heartbeat,
471 _ = ticker.tick() => {
472 snapshot_update(&snapshot, |current| {
473 current.state = ApplicationHeartbeatState::Running;
474 });
475 let options = match send_options(config.send_timeout) {
476 Ok(options) => options,
477 Err(error) => {
478 diagnostics.error(session_id, HeartbeatPhase::SendOptions, &error);
479 consecutive_missed = 0;
483 let should_stop = record_failure(
484 &snapshot,
485 &error,
486 config.failure_action,
487 );
488 if should_stop {
489 stopped_by_policy = true;
490 break 'heartbeat;
491 }
492 continue;
493 }
494 };
495 let sent_at = SystemTime::now();
496 let send = sender
497 .message(config.payload.clone())
498 .options(options)
499 .send();
500 let send_result = tokio::select! {
501 _ = cancellation.cancelled() => break 'heartbeat,
502 _ = session_closed.cancelled() => break 'heartbeat,
503 result = send => result,
504 };
505 if let Err(error) = send_result {
506 diagnostics.error(session_id, HeartbeatPhase::Send, &error);
507 consecutive_missed = 0;
510 let should_stop = record_failure(
511 &snapshot,
512 &error,
513 config.failure_action,
514 );
515 if should_stop {
516 stopped_by_policy = true;
517 break 'heartbeat;
518 }
519 continue;
520 }
521 snapshot_update(&snapshot, |current| {
522 increment(&mut current.sent);
523 });
524 let (Some(matcher), Some(receiver)) =
525 (config.matcher.as_ref(), receiver.as_mut())
526 else {
527 diagnostics.recovered(session_id);
528 continue;
529 };
530 let connection_id = match runtime.current_connection_id() {
531 Ok(Some(connection_id)) => connection_id,
532 Ok(None) => {
533 consecutive_missed = 0;
538 let error = NetError::from(ErrorKind::NotConnected)
539 .with_stage(ErrorStage::Runtime);
540 let should_stop = record_failure(
541 &snapshot,
542 &error,
543 config.failure_action,
544 );
545 if should_stop {
546 diagnostics.last = Some(HeartbeatFailure {
549 phase: HeartbeatPhase::Connection,
550 kind: ErrorKind::NotConnected,
551 stage: Some(ErrorStage::Runtime),
552 });
553 diagnostics.failures = diagnostics.failures.saturating_add(1);
554 stopped_by_policy = true;
555 break 'heartbeat;
556 }
557 continue;
558 }
559 Err(error) => {
560 diagnostics.error(session_id, HeartbeatPhase::Connection, &error);
561 consecutive_missed = 0;
565 let should_stop = record_failure(
566 &snapshot,
567 &error,
568 config.failure_action,
569 );
570 if should_stop {
571 stopped_by_policy = true;
572 break 'heartbeat;
573 }
574 continue;
575 }
576 };
577 snapshot_update(&snapshot, |current| {
578 current.state = ApplicationHeartbeatState::WaitingForResponse;
579 });
580 let timeout = match config.response_timeout {
581 Some(timeout) => timeout,
582 None => continue,
583 };
584 match wait_for_response(
585 receiver,
586 matcher,
587 timeout,
588 &cancellation,
589 &session_closed,
590 Some(connection_id),
591 sent_at,
592 ).await {
593 ResponseWait::Matched => {
594 consecutive_missed = 0;
595 snapshot_update(&snapshot, |current| {
596 increment(&mut current.matched_responses);
597 current.state = ApplicationHeartbeatState::Running;
598 });
599 diagnostics.recovered(session_id);
600 }
601 ResponseWait::TimedOut => {
602 diagnostics.failed(session_id, HeartbeatFailure {
603 phase: HeartbeatPhase::ResponseWait, kind: ErrorKind::TimedOut, stage: Some(ErrorStage::Receive),
604 });
605 consecutive_missed = consecutive_missed.saturating_add(1);
606 snapshot_update(&snapshot, |current| {
607 increment(&mut current.missed_responses);
608 current.last_error = Some(ErrorKind::TimedOut);
609 increment(&mut current.failures);
610 current.state = ApplicationHeartbeatState::Running;
611 });
612 if config.failure_action == ApplicationHeartbeatFailureAction::Stop
613 && consecutive_missed >= config.max_missed_responses
614 {
615 stopped_by_policy = true;
616 break 'heartbeat;
617 }
618 }
619 ResponseWait::Closed | ResponseWait::Cancelled => break 'heartbeat,
620 ResponseWait::Failed(failure) => {
621 diagnostics.failed(session_id, failure);
622 consecutive_missed = 0;
625 let should_stop = record_failure_kind(
626 &snapshot,
627 failure.kind,
628 config.failure_action,
629 );
630 if should_stop {
631 stopped_by_policy = true;
632 break 'heartbeat;
633 }
634 }
635 }
636 }
637 }
638 }
639 if let Some(receiver) = receiver.as_ref() {
640 receiver.unsubscribe();
641 }
642 snapshot_update(&snapshot, |current| {
643 current.state = ApplicationHeartbeatState::Stopped;
644 });
645 if stopped_by_policy {
646 diagnostics.stopped(session_id);
647 }
648}
649
650async fn wait_for_response(
651 receiver: &mut MessageReceiver,
652 matcher: &ApplicationHeartbeatMatcher,
653 timeout: Duration,
654 cancellation: &CancellationToken,
655 session_closed: &CancellationToken,
656 connection_id: Option<crate::ws::ConnectionId>,
657 sent_at: SystemTime,
658) -> ResponseWait {
659 let deadline = time::sleep(timeout);
660 tokio::pin!(deadline);
661 loop {
662 tokio::select! {
663 _ = cancellation.cancelled() => return ResponseWait::Cancelled,
664 _ = session_closed.cancelled() => return ResponseWait::Closed,
665 _ = &mut deadline => return ResponseWait::TimedOut,
666 result = receiver.recv() => {
667 match result {
668 Ok(Some(message)) => {
669 if connection_id != Some(message.connection_id()) {
670 continue;
671 }
672 if message.received_at().duration_since(sent_at).is_err() {
673 continue;
674 }
675 let matches = match catch_unwind(AssertUnwindSafe(|| matcher(&message))) {
676 Ok(matches) => matches,
677 Err(_) => return ResponseWait::Failed(HeartbeatFailure {
678 phase: HeartbeatPhase::Matcher, kind: ErrorKind::CallbackPanicked, stage: Some(ErrorStage::Receive),
679 }),
680 };
681 if matches {
682 return ResponseWait::Matched;
683 }
684 }
685 Ok(None) => return ResponseWait::Closed,
686 Err(ReceiveError::Lagged { .. }) => continue,
692 Err(ReceiveError::Failed(error)) => {
693 return ResponseWait::Failed(HeartbeatFailure {
694 phase: HeartbeatPhase::Receive, kind: error.kind(), stage: error.context().stage,
695 });
696 }
697 }
698 }
699 }
700 }
701}
702
703fn send_options(timeout: Duration) -> Result<SendOptions> {
704 let deadline = StdInstant::now().checked_add(timeout).ok_or_else(|| {
705 NetError::config(
706 "application_heartbeat.send_timeout",
707 "cannot be represented as a deadline",
708 )
709 })?;
710 Ok(SendOptions {
711 disconnected: DisconnectedPolicy::WaitForReconnect,
712 enqueue_timeout: Some(timeout),
713 write_timeout: timeout,
714 deadline: Some(deadline),
715 ..SendOptions::default()
716 })
717}
718
719fn record_failure(
720 snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
721 error: &NetError,
722 action: ApplicationHeartbeatFailureAction,
723) -> bool {
724 record_failure_kind(snapshot, error.kind(), action)
725}
726
727fn record_failure_kind(
728 snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
729 kind: ErrorKind,
730 action: ApplicationHeartbeatFailureAction,
731) -> bool {
732 snapshot_update(snapshot, |current| {
733 increment(&mut current.failures);
734 current.last_error = Some(kind);
735 current.state = ApplicationHeartbeatState::Running;
736 });
737 action == ApplicationHeartbeatFailureAction::Stop
738}
739
740fn increment(value: &mut u64) {
741 if let Some(next) = value.checked_add(1) {
742 *value = next;
743 }
744}
745
746fn snapshot_copy(
747 snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
748) -> ApplicationHeartbeatSnapshot {
749 *lock_snapshot(snapshot)
750}
751
752fn snapshot_update<F>(snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>, update: F)
753where
754 F: FnOnce(&mut ApplicationHeartbeatSnapshot),
755{
756 let mut current = lock_snapshot(snapshot);
757 update(&mut current);
758}
759
760fn lock_snapshot<'a>(
761 snapshot: &'a Arc<Mutex<ApplicationHeartbeatSnapshot>>,
762) -> MutexGuard<'a, ApplicationHeartbeatSnapshot> {
763 match snapshot.lock() {
764 Ok(guard) => guard,
765 Err(poisoned) => {
766 crate::log_e!(crate::common::log::log_def::LogType::WSC;
767 "ws.heartbeat.lock_snapshot", "arg_snapshot_present|error", true, "lock_poisoned_recovered");
768 poisoned.into_inner()
769 }
770 }
771}
772
773#[cfg(test)]
774#[path = "application_heartbeat_tests.rs"]
775mod tests;