1use super::{
8 DisconnectedPolicy, IncomingMessage, Message, MessageReceiver, ReceiveOptions, ReceiveOverflow,
9 SendOptions, Sender,
10};
11use crate::error::{ErrorKind, ErrorStage, ReceiveError};
12use crate::module::ws_client::session_runtime::SessionRuntime;
13use crate::{NetError, Result};
14use std::panic::{catch_unwind, AssertUnwindSafe};
15use std::sync::{Arc, Mutex, MutexGuard};
16use std::time::{Duration, Instant as StdInstant, SystemTime};
17use tokio::task::JoinHandle;
18use tokio::time::{self, Instant};
19use tokio_util::sync::CancellationToken;
20
21pub type ApplicationHeartbeatMatcher =
29 Arc<dyn Fn(&IncomingMessage) -> bool + Send + Sync + 'static>;
30
31#[derive(Clone, Copy, Debug, Eq, PartialEq)]
33pub enum ApplicationHeartbeatFailureAction {
34 Continue,
36 Stop,
38}
39
40#[derive(Clone)]
42pub struct ApplicationHeartbeatConfig {
43 pub interval: Duration,
45 pub payload: Message,
47 pub send_timeout: Duration,
49 pub response_timeout: Option<Duration>,
52 pub max_missed_responses: u32,
54 pub matcher: Option<ApplicationHeartbeatMatcher>,
56 pub receive_options: ReceiveOptions,
61 pub failure_action: ApplicationHeartbeatFailureAction,
63}
64
65impl ApplicationHeartbeatConfig {
66 pub fn new(interval: Duration, payload: impl Into<Message>) -> Self {
68 Self {
69 interval,
70 payload: payload.into(),
71 send_timeout: Duration::from_secs(10),
72 response_timeout: None,
73 max_missed_responses: 1,
74 matcher: None,
75 receive_options: ReceiveOptions {
76 overflow: ReceiveOverflow::DropOldest,
77 ..ReceiveOptions::default()
78 },
79 failure_action: ApplicationHeartbeatFailureAction::Continue,
80 }
81 }
82
83 pub fn with_send_timeout(mut self, timeout: Duration) -> Self {
85 self.send_timeout = timeout;
86 self
87 }
88
89 pub fn with_response_matcher(
91 mut self,
92 timeout: Duration,
93 matcher: impl Fn(&IncomingMessage) -> bool + Send + Sync + 'static,
94 ) -> Self {
95 self.response_timeout = Some(timeout);
96 self.matcher = Some(Arc::new(matcher));
97 self
98 }
99
100 pub fn with_max_missed_responses(mut self, max_missed_responses: u32) -> Self {
102 self.max_missed_responses = max_missed_responses;
103 self
104 }
105
106 pub fn with_receive_options(mut self, receive_options: ReceiveOptions) -> Self {
108 self.receive_options = receive_options;
109 self
110 }
111
112 pub fn with_failure_action(
114 mut self,
115 failure_action: ApplicationHeartbeatFailureAction,
116 ) -> Self {
117 self.failure_action = failure_action;
118 self
119 }
120
121 pub fn validate(&self) -> Result<()> {
123 if self.interval.is_zero() {
124 return Err(NetError::config(
125 "application_heartbeat.interval",
126 "must be greater than zero",
127 ));
128 }
129 if Instant::now().checked_add(self.interval).is_none() {
130 return Err(NetError::config(
131 "application_heartbeat.interval",
132 "cannot be represented as a timer deadline",
133 ));
134 }
135 if self.send_timeout.is_zero() {
136 return Err(NetError::config(
137 "application_heartbeat.send_timeout",
138 "must be greater than zero",
139 ));
140 }
141 if StdInstant::now().checked_add(self.send_timeout).is_none() {
142 return Err(NetError::config(
143 "application_heartbeat.send_timeout",
144 "cannot be represented as a deadline",
145 ));
146 }
147 if self.max_missed_responses == 0 {
148 return Err(NetError::config(
149 "application_heartbeat.max_missed_responses",
150 "must be greater than zero",
151 ));
152 }
153 if self.matcher.is_some() && self.receive_options.include_control_frames {
154 return Err(NetError::config(
155 "application_heartbeat.receive_options.include_control_frames",
156 "must be false so control Pong frames cannot acknowledge application heartbeats",
157 ));
158 }
159 if self.matcher.is_some() && self.response_timeout.is_none() {
160 return Err(NetError::config(
161 "application_heartbeat.response_timeout",
162 "is required when matcher is set",
163 ));
164 }
165 if let Some(timeout) = self.response_timeout {
166 if StdInstant::now().checked_add(timeout).is_none() {
167 return Err(NetError::config(
168 "application_heartbeat.response_timeout",
169 "must fit an absolute deadline",
170 ));
171 }
172 if timeout.is_zero() {
173 return Err(NetError::config(
174 "application_heartbeat.response_timeout",
175 "must be greater than zero",
176 ));
177 }
178 if Instant::now().checked_add(timeout).is_none() {
179 return Err(NetError::config(
180 "application_heartbeat.response_timeout",
181 "cannot be represented as a deadline",
182 ));
183 }
184 if self.matcher.is_none() {
185 return Err(NetError::config(
186 "application_heartbeat.matcher",
187 "is required when response_timeout is set",
188 ));
189 }
190 }
191 if self.matcher.is_some() {
192 self.receive_options.validate()?;
193 }
194 Ok(())
195 }
196}
197
198impl std::fmt::Debug for ApplicationHeartbeatConfig {
199 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
200 formatter
201 .debug_struct("ApplicationHeartbeatConfig")
202 .field("interval", &self.interval)
203 .field("payload_bytes", &self.payload.len())
204 .field("send_timeout", &self.send_timeout)
205 .field("response_timeout", &self.response_timeout)
206 .field("max_missed_responses", &self.max_missed_responses)
207 .field("has_matcher", &self.matcher.is_some())
208 .field("receive_options", &self.receive_options)
209 .field("failure_action", &self.failure_action)
210 .finish()
211 }
212}
213
214#[derive(Clone, Copy, Debug, Eq, PartialEq)]
216pub enum ApplicationHeartbeatState {
217 Running,
219 WaitingForResponse,
221 Stopped,
223}
224
225#[derive(Clone, Copy, Debug, Eq, PartialEq)]
227pub struct ApplicationHeartbeatSnapshot {
228 pub state: ApplicationHeartbeatState,
230 pub sent: u64,
232 pub matched_responses: u64,
234 pub missed_responses: u64,
236 pub failures: u64,
238 pub last_error: Option<ErrorKind>,
240}
241
242impl Default for ApplicationHeartbeatSnapshot {
243 fn default() -> Self {
244 Self {
245 state: ApplicationHeartbeatState::Running,
246 sent: 0,
247 matched_responses: 0,
248 missed_responses: 0,
249 failures: 0,
250 last_error: None,
251 }
252 }
253}
254
255pub struct ApplicationHeartbeatHandle {
260 cancellation: CancellationToken,
261 snapshot: Arc<Mutex<ApplicationHeartbeatSnapshot>>,
262 task: Option<JoinHandle<()>>,
263}
264
265impl ApplicationHeartbeatHandle {
266 pub(crate) fn new(
267 sender: Sender,
268 receiver: Option<MessageReceiver>,
269 config: ApplicationHeartbeatConfig,
270 ) -> Result<Self> {
271 config.validate()?;
272 let cancellation = CancellationToken::new();
273 let snapshot = Arc::new(Mutex::new(ApplicationHeartbeatSnapshot::default()));
274 let runtime = sender.runtime.clone();
275 let task = runtime.clone().spawn(run_heartbeat(
276 sender,
277 receiver,
278 config,
279 cancellation.clone(),
280 snapshot.clone(),
281 runtime,
282 ))?;
283 Ok(Self {
284 cancellation,
285 snapshot,
286 task: Some(task),
287 })
288 }
289
290 pub fn stop(&self) {
292 self.cancellation.cancel();
293 }
294
295 pub fn snapshot(&self) -> ApplicationHeartbeatSnapshot {
297 snapshot_copy(&self.snapshot)
298 }
299
300 pub async fn stopped(mut self) -> Result<ApplicationHeartbeatSnapshot> {
302 let task = self
303 .task
304 .take()
305 .ok_or_else(|| NetError::from(ErrorKind::Internal).with_stage(ErrorStage::Runtime))?;
306 task.await.map_err(|error| {
307 NetError::with_source(ErrorKind::EngineDropped, error).with_stage(ErrorStage::Runtime)
308 })?;
309 Ok(snapshot_copy(&self.snapshot))
310 }
311}
312
313impl Drop for ApplicationHeartbeatHandle {
314 fn drop(&mut self) {
315 self.cancellation.cancel();
316 if let Some(task) = self.task.take() {
317 task.abort();
318 }
319 }
320}
321
322#[derive(Clone, Copy, Debug, Eq, PartialEq)]
323enum ResponseWait {
324 Matched,
325 TimedOut,
326 Closed,
327 Cancelled,
328 Failed(ErrorKind),
329}
330
331async fn run_heartbeat(
332 sender: Sender,
333 mut receiver: Option<MessageReceiver>,
334 config: ApplicationHeartbeatConfig,
335 cancellation: CancellationToken,
336 snapshot: Arc<Mutex<ApplicationHeartbeatSnapshot>>,
337 runtime: Arc<SessionRuntime>,
338) {
339 let now = Instant::now();
340 let first_tick = match now.checked_add(config.interval) {
341 Some(deadline) => deadline,
342 None => now,
343 };
344 let mut ticker = time::interval_at(first_tick, config.interval);
345 ticker.set_missed_tick_behavior(time::MissedTickBehavior::Skip);
346 let mut consecutive_missed = 0_u32;
347 let session_closed = runtime.closed_token();
348
349 'heartbeat: loop {
350 tokio::select! {
351 _ = cancellation.cancelled() => break 'heartbeat,
352 _ = session_closed.cancelled() => break 'heartbeat,
353 _ = ticker.tick() => {
354 snapshot_update(&snapshot, |current| {
355 current.state = ApplicationHeartbeatState::Running;
356 });
357 let options = match send_options(config.send_timeout) {
358 Ok(options) => options,
359 Err(error) => {
360 consecutive_missed = 0;
364 let should_stop = record_failure(
365 &snapshot,
366 &error,
367 config.failure_action,
368 );
369 if should_stop {
370 break 'heartbeat;
371 }
372 continue;
373 }
374 };
375 let sent_at = SystemTime::now();
376 let send = sender
377 .message(config.payload.clone())
378 .options(options)
379 .send();
380 let send_result = tokio::select! {
381 _ = cancellation.cancelled() => break 'heartbeat,
382 _ = session_closed.cancelled() => break 'heartbeat,
383 result = send => result,
384 };
385 if let Err(error) = send_result {
386 consecutive_missed = 0;
389 let should_stop = record_failure(
390 &snapshot,
391 &error,
392 config.failure_action,
393 );
394 if should_stop {
395 break 'heartbeat;
396 }
397 continue;
398 }
399 snapshot_update(&snapshot, |current| {
400 increment(&mut current.sent);
401 });
402 let (Some(matcher), Some(receiver)) =
403 (config.matcher.as_ref(), receiver.as_mut())
404 else {
405 continue;
406 };
407 let connection_id = match runtime.current_connection_id() {
408 Ok(Some(connection_id)) => connection_id,
409 Ok(None) => {
410 consecutive_missed = 0;
415 let error = NetError::from(ErrorKind::NotConnected)
416 .with_stage(ErrorStage::Runtime);
417 let should_stop = record_failure(
418 &snapshot,
419 &error,
420 config.failure_action,
421 );
422 if should_stop {
423 break 'heartbeat;
424 }
425 continue;
426 }
427 Err(error) => {
428 consecutive_missed = 0;
432 let should_stop = record_failure(
433 &snapshot,
434 &error,
435 config.failure_action,
436 );
437 if should_stop {
438 break 'heartbeat;
439 }
440 continue;
441 }
442 };
443 snapshot_update(&snapshot, |current| {
444 current.state = ApplicationHeartbeatState::WaitingForResponse;
445 });
446 let timeout = match config.response_timeout {
447 Some(timeout) => timeout,
448 None => continue,
449 };
450 match wait_for_response(
451 receiver,
452 matcher,
453 timeout,
454 &cancellation,
455 &session_closed,
456 Some(connection_id),
457 sent_at,
458 ).await {
459 ResponseWait::Matched => {
460 consecutive_missed = 0;
461 snapshot_update(&snapshot, |current| {
462 increment(&mut current.matched_responses);
463 current.state = ApplicationHeartbeatState::Running;
464 });
465 }
466 ResponseWait::TimedOut => {
467 consecutive_missed = consecutive_missed.saturating_add(1);
468 snapshot_update(&snapshot, |current| {
469 increment(&mut current.missed_responses);
470 current.last_error = Some(ErrorKind::TimedOut);
471 increment(&mut current.failures);
472 current.state = ApplicationHeartbeatState::Running;
473 });
474 if config.failure_action == ApplicationHeartbeatFailureAction::Stop
475 && consecutive_missed >= config.max_missed_responses
476 {
477 break 'heartbeat;
478 }
479 }
480 ResponseWait::Closed | ResponseWait::Cancelled => break 'heartbeat,
481 ResponseWait::Failed(kind) => {
482 consecutive_missed = 0;
485 let should_stop = record_failure_kind(
486 &snapshot,
487 kind,
488 config.failure_action,
489 );
490 if should_stop {
491 break 'heartbeat;
492 }
493 }
494 }
495 }
496 }
497 }
498 if let Some(receiver) = receiver.as_ref() {
499 receiver.unsubscribe();
500 }
501 snapshot_update(&snapshot, |current| {
502 current.state = ApplicationHeartbeatState::Stopped;
503 });
504}
505
506async fn wait_for_response(
507 receiver: &mut MessageReceiver,
508 matcher: &ApplicationHeartbeatMatcher,
509 timeout: Duration,
510 cancellation: &CancellationToken,
511 session_closed: &CancellationToken,
512 connection_id: Option<crate::ws::ConnectionId>,
513 sent_at: SystemTime,
514) -> ResponseWait {
515 let deadline = time::sleep(timeout);
516 tokio::pin!(deadline);
517 loop {
518 tokio::select! {
519 _ = cancellation.cancelled() => return ResponseWait::Cancelled,
520 _ = session_closed.cancelled() => return ResponseWait::Closed,
521 _ = &mut deadline => return ResponseWait::TimedOut,
522 result = receiver.recv() => {
523 match result {
524 Ok(Some(message)) => {
525 if connection_id != Some(message.connection_id()) {
526 continue;
527 }
528 if message.received_at().duration_since(sent_at).is_err() {
529 continue;
530 }
531 let matches = match catch_unwind(AssertUnwindSafe(|| matcher(&message))) {
532 Ok(matches) => matches,
533 Err(_) => return ResponseWait::Failed(ErrorKind::CallbackPanicked),
534 };
535 if matches {
536 return ResponseWait::Matched;
537 }
538 }
539 Ok(None) => return ResponseWait::Closed,
540 Err(ReceiveError::Lagged { .. }) => continue,
546 Err(ReceiveError::Failed(error)) => {
547 return ResponseWait::Failed(error.kind());
548 }
549 }
550 }
551 }
552 }
553}
554
555fn send_options(timeout: Duration) -> Result<SendOptions> {
556 let deadline = StdInstant::now().checked_add(timeout).ok_or_else(|| {
557 NetError::config(
558 "application_heartbeat.send_timeout",
559 "cannot be represented as a deadline",
560 )
561 })?;
562 Ok(SendOptions {
563 disconnected: DisconnectedPolicy::WaitForReconnect,
564 enqueue_timeout: Some(timeout),
565 write_timeout: timeout,
566 deadline: Some(deadline),
567 ..SendOptions::default()
568 })
569}
570
571fn record_failure(
572 snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
573 error: &NetError,
574 action: ApplicationHeartbeatFailureAction,
575) -> bool {
576 record_failure_kind(snapshot, error.kind(), action)
577}
578
579fn record_failure_kind(
580 snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
581 kind: ErrorKind,
582 action: ApplicationHeartbeatFailureAction,
583) -> bool {
584 snapshot_update(snapshot, |current| {
585 increment(&mut current.failures);
586 current.last_error = Some(kind);
587 current.state = ApplicationHeartbeatState::Running;
588 });
589 action == ApplicationHeartbeatFailureAction::Stop
590}
591
592fn increment(value: &mut u64) {
593 if let Some(next) = value.checked_add(1) {
594 *value = next;
595 }
596}
597
598fn snapshot_copy(
599 snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
600) -> ApplicationHeartbeatSnapshot {
601 *lock_snapshot(snapshot)
602}
603
604fn snapshot_update<F>(snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>, update: F)
605where
606 F: FnOnce(&mut ApplicationHeartbeatSnapshot),
607{
608 let mut current = lock_snapshot(snapshot);
609 update(&mut current);
610}
611
612fn lock_snapshot<'a>(
613 snapshot: &'a Arc<Mutex<ApplicationHeartbeatSnapshot>>,
614) -> MutexGuard<'a, ApplicationHeartbeatSnapshot> {
615 match snapshot.lock() {
616 Ok(guard) => guard,
617 Err(poisoned) => {
618 crate::log_e!(crate::common::log::log_def::LogType::WSC;
619 "application_heartbeat_snapshot", "error", "lock_poisoned_recovered");
620 poisoned.into_inner()
621 }
622 }
623}
624
625#[cfg(test)]
626#[path = "application_heartbeat_tests.rs"]
627mod tests;