use super::{
DisconnectedPolicy, IncomingMessage, Message, MessageReceiver, ReceiveOptions, ReceiveOverflow,
SendOptions, Sender,
};
use crate::error::{ErrorKind, ErrorStage, ReceiveError};
use crate::module::ws_client::session_runtime::SessionRuntime;
use crate::{NetError, Result};
use std::panic::{catch_unwind, AssertUnwindSafe};
use std::sync::{Arc, Mutex, MutexGuard};
use std::time::{Duration, Instant as StdInstant, SystemTime};
use tokio::task::JoinHandle;
use tokio::time::{self, Instant};
use tokio_util::sync::CancellationToken;
pub type ApplicationHeartbeatMatcher =
Arc<dyn Fn(&IncomingMessage) -> bool + Send + Sync + 'static>;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ApplicationHeartbeatFailureAction {
Continue,
Stop,
}
#[derive(Clone)]
pub struct ApplicationHeartbeatConfig {
pub interval: Duration,
pub payload: Message,
pub send_timeout: Duration,
pub response_timeout: Option<Duration>,
pub max_missed_responses: u32,
pub matcher: Option<ApplicationHeartbeatMatcher>,
pub receive_options: ReceiveOptions,
pub failure_action: ApplicationHeartbeatFailureAction,
}
impl ApplicationHeartbeatConfig {
pub fn new(interval: Duration, payload: impl Into<Message>) -> Self {
Self {
interval,
payload: payload.into(),
send_timeout: Duration::from_secs(10),
response_timeout: None,
max_missed_responses: 1,
matcher: None,
receive_options: ReceiveOptions {
overflow: ReceiveOverflow::DropOldest,
..ReceiveOptions::default()
},
failure_action: ApplicationHeartbeatFailureAction::Continue,
}
}
pub fn with_send_timeout(mut self, timeout: Duration) -> Self {
self.send_timeout = timeout;
self
}
pub fn with_response_matcher(
mut self,
timeout: Duration,
matcher: impl Fn(&IncomingMessage) -> bool + Send + Sync + 'static,
) -> Self {
self.response_timeout = Some(timeout);
self.matcher = Some(Arc::new(matcher));
self
}
pub fn with_max_missed_responses(mut self, max_missed_responses: u32) -> Self {
self.max_missed_responses = max_missed_responses;
self
}
pub fn with_receive_options(mut self, receive_options: ReceiveOptions) -> Self {
self.receive_options = receive_options;
self
}
pub fn with_failure_action(
mut self,
failure_action: ApplicationHeartbeatFailureAction,
) -> Self {
self.failure_action = failure_action;
self
}
pub fn validate(&self) -> Result<()> {
if self.interval.is_zero() {
return Err(NetError::config(
"application_heartbeat.interval",
"must be greater than zero",
));
}
if Instant::now().checked_add(self.interval).is_none() {
return Err(NetError::config(
"application_heartbeat.interval",
"cannot be represented as a timer deadline",
));
}
if self.send_timeout.is_zero() {
return Err(NetError::config(
"application_heartbeat.send_timeout",
"must be greater than zero",
));
}
if StdInstant::now().checked_add(self.send_timeout).is_none() {
return Err(NetError::config(
"application_heartbeat.send_timeout",
"cannot be represented as a deadline",
));
}
if self.max_missed_responses == 0 {
return Err(NetError::config(
"application_heartbeat.max_missed_responses",
"must be greater than zero",
));
}
if self.matcher.is_some() && self.receive_options.include_control_frames {
return Err(NetError::config(
"application_heartbeat.receive_options.include_control_frames",
"must be false so control Pong frames cannot acknowledge application heartbeats",
));
}
if self.matcher.is_some() && self.response_timeout.is_none() {
return Err(NetError::config(
"application_heartbeat.response_timeout",
"is required when matcher is set",
));
}
if let Some(timeout) = self.response_timeout {
if StdInstant::now().checked_add(timeout).is_none() {
return Err(NetError::config(
"application_heartbeat.response_timeout",
"must fit an absolute deadline",
));
}
if timeout.is_zero() {
return Err(NetError::config(
"application_heartbeat.response_timeout",
"must be greater than zero",
));
}
if Instant::now().checked_add(timeout).is_none() {
return Err(NetError::config(
"application_heartbeat.response_timeout",
"cannot be represented as a deadline",
));
}
if self.matcher.is_none() {
return Err(NetError::config(
"application_heartbeat.matcher",
"is required when response_timeout is set",
));
}
}
if self.matcher.is_some() {
self.receive_options.validate()?;
}
Ok(())
}
}
impl std::fmt::Debug for ApplicationHeartbeatConfig {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
.debug_struct("ApplicationHeartbeatConfig")
.field("interval", &self.interval)
.field("payload_bytes", &self.payload.len())
.field("send_timeout", &self.send_timeout)
.field("response_timeout", &self.response_timeout)
.field("max_missed_responses", &self.max_missed_responses)
.field("has_matcher", &self.matcher.is_some())
.field("receive_options", &self.receive_options)
.field("failure_action", &self.failure_action)
.finish()
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ApplicationHeartbeatState {
Running,
WaitingForResponse,
Stopped,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct ApplicationHeartbeatSnapshot {
pub state: ApplicationHeartbeatState,
pub sent: u64,
pub matched_responses: u64,
pub missed_responses: u64,
pub failures: u64,
pub last_error: Option<ErrorKind>,
}
impl Default for ApplicationHeartbeatSnapshot {
fn default() -> Self {
Self {
state: ApplicationHeartbeatState::Running,
sent: 0,
matched_responses: 0,
missed_responses: 0,
failures: 0,
last_error: None,
}
}
}
pub struct ApplicationHeartbeatHandle {
cancellation: CancellationToken,
snapshot: Arc<Mutex<ApplicationHeartbeatSnapshot>>,
task: Option<JoinHandle<()>>,
}
impl ApplicationHeartbeatHandle {
pub(crate) fn new(
sender: Sender,
receiver: Option<MessageReceiver>,
config: ApplicationHeartbeatConfig,
) -> Result<Self> {
config.validate()?;
let cancellation = CancellationToken::new();
let snapshot = Arc::new(Mutex::new(ApplicationHeartbeatSnapshot::default()));
let runtime = sender.runtime.clone();
let task = runtime.clone().spawn(run_heartbeat(
sender,
receiver,
config,
cancellation.clone(),
snapshot.clone(),
runtime,
))?;
Ok(Self {
cancellation,
snapshot,
task: Some(task),
})
}
pub fn stop(&self) {
self.cancellation.cancel();
}
pub fn snapshot(&self) -> ApplicationHeartbeatSnapshot {
snapshot_copy(&self.snapshot)
}
pub async fn stopped(mut self) -> Result<ApplicationHeartbeatSnapshot> {
let task = self
.task
.take()
.ok_or_else(|| NetError::from(ErrorKind::Internal).with_stage(ErrorStage::Runtime))?;
task.await.map_err(|error| {
NetError::with_source(ErrorKind::EngineDropped, error).with_stage(ErrorStage::Runtime)
})?;
Ok(snapshot_copy(&self.snapshot))
}
}
impl Drop for ApplicationHeartbeatHandle {
fn drop(&mut self) {
self.cancellation.cancel();
if let Some(task) = self.task.take() {
task.abort();
}
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum ResponseWait {
Matched,
TimedOut,
Closed,
Cancelled,
Failed(ErrorKind),
}
async fn run_heartbeat(
sender: Sender,
mut receiver: Option<MessageReceiver>,
config: ApplicationHeartbeatConfig,
cancellation: CancellationToken,
snapshot: Arc<Mutex<ApplicationHeartbeatSnapshot>>,
runtime: Arc<SessionRuntime>,
) {
let now = Instant::now();
let first_tick = match now.checked_add(config.interval) {
Some(deadline) => deadline,
None => now,
};
let mut ticker = time::interval_at(first_tick, config.interval);
ticker.set_missed_tick_behavior(time::MissedTickBehavior::Skip);
let mut consecutive_missed = 0_u32;
let session_closed = runtime.closed_token();
'heartbeat: loop {
tokio::select! {
_ = cancellation.cancelled() => break 'heartbeat,
_ = session_closed.cancelled() => break 'heartbeat,
_ = ticker.tick() => {
snapshot_update(&snapshot, |current| {
current.state = ApplicationHeartbeatState::Running;
});
let options = match send_options(config.send_timeout) {
Ok(options) => options,
Err(error) => {
consecutive_missed = 0;
let should_stop = record_failure(
&snapshot,
&error,
config.failure_action,
);
if should_stop {
break 'heartbeat;
}
continue;
}
};
let sent_at = SystemTime::now();
let send = sender
.message(config.payload.clone())
.options(options)
.send();
let send_result = tokio::select! {
_ = cancellation.cancelled() => break 'heartbeat,
_ = session_closed.cancelled() => break 'heartbeat,
result = send => result,
};
if let Err(error) = send_result {
consecutive_missed = 0;
let should_stop = record_failure(
&snapshot,
&error,
config.failure_action,
);
if should_stop {
break 'heartbeat;
}
continue;
}
snapshot_update(&snapshot, |current| {
increment(&mut current.sent);
});
let (Some(matcher), Some(receiver)) =
(config.matcher.as_ref(), receiver.as_mut())
else {
continue;
};
let connection_id = match runtime.current_connection_id() {
Ok(Some(connection_id)) => connection_id,
Ok(None) => {
consecutive_missed = 0;
let error = NetError::from(ErrorKind::NotConnected)
.with_stage(ErrorStage::Runtime);
let should_stop = record_failure(
&snapshot,
&error,
config.failure_action,
);
if should_stop {
break 'heartbeat;
}
continue;
}
Err(error) => {
consecutive_missed = 0;
let should_stop = record_failure(
&snapshot,
&error,
config.failure_action,
);
if should_stop {
break 'heartbeat;
}
continue;
}
};
snapshot_update(&snapshot, |current| {
current.state = ApplicationHeartbeatState::WaitingForResponse;
});
let timeout = match config.response_timeout {
Some(timeout) => timeout,
None => continue,
};
match wait_for_response(
receiver,
matcher,
timeout,
&cancellation,
&session_closed,
Some(connection_id),
sent_at,
).await {
ResponseWait::Matched => {
consecutive_missed = 0;
snapshot_update(&snapshot, |current| {
increment(&mut current.matched_responses);
current.state = ApplicationHeartbeatState::Running;
});
}
ResponseWait::TimedOut => {
consecutive_missed = consecutive_missed.saturating_add(1);
snapshot_update(&snapshot, |current| {
increment(&mut current.missed_responses);
current.last_error = Some(ErrorKind::TimedOut);
increment(&mut current.failures);
current.state = ApplicationHeartbeatState::Running;
});
if config.failure_action == ApplicationHeartbeatFailureAction::Stop
&& consecutive_missed >= config.max_missed_responses
{
break 'heartbeat;
}
}
ResponseWait::Closed | ResponseWait::Cancelled => break 'heartbeat,
ResponseWait::Failed(kind) => {
consecutive_missed = 0;
let should_stop = record_failure_kind(
&snapshot,
kind,
config.failure_action,
);
if should_stop {
break 'heartbeat;
}
}
}
}
}
}
if let Some(receiver) = receiver.as_ref() {
receiver.unsubscribe();
}
snapshot_update(&snapshot, |current| {
current.state = ApplicationHeartbeatState::Stopped;
});
}
async fn wait_for_response(
receiver: &mut MessageReceiver,
matcher: &ApplicationHeartbeatMatcher,
timeout: Duration,
cancellation: &CancellationToken,
session_closed: &CancellationToken,
connection_id: Option<crate::ws::ConnectionId>,
sent_at: SystemTime,
) -> ResponseWait {
let deadline = time::sleep(timeout);
tokio::pin!(deadline);
loop {
tokio::select! {
_ = cancellation.cancelled() => return ResponseWait::Cancelled,
_ = session_closed.cancelled() => return ResponseWait::Closed,
_ = &mut deadline => return ResponseWait::TimedOut,
result = receiver.recv() => {
match result {
Ok(Some(message)) => {
if connection_id != Some(message.connection_id()) {
continue;
}
if message.received_at().duration_since(sent_at).is_err() {
continue;
}
let matches = match catch_unwind(AssertUnwindSafe(|| matcher(&message))) {
Ok(matches) => matches,
Err(_) => return ResponseWait::Failed(ErrorKind::CallbackPanicked),
};
if matches {
return ResponseWait::Matched;
}
}
Ok(None) => return ResponseWait::Closed,
Err(ReceiveError::Lagged { .. }) => continue,
Err(ReceiveError::Failed(error)) => {
return ResponseWait::Failed(error.kind());
}
}
}
}
}
}
fn send_options(timeout: Duration) -> Result<SendOptions> {
let deadline = StdInstant::now().checked_add(timeout).ok_or_else(|| {
NetError::config(
"application_heartbeat.send_timeout",
"cannot be represented as a deadline",
)
})?;
Ok(SendOptions {
disconnected: DisconnectedPolicy::WaitForReconnect,
enqueue_timeout: Some(timeout),
write_timeout: timeout,
deadline: Some(deadline),
..SendOptions::default()
})
}
fn record_failure(
snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
error: &NetError,
action: ApplicationHeartbeatFailureAction,
) -> bool {
record_failure_kind(snapshot, error.kind(), action)
}
fn record_failure_kind(
snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
kind: ErrorKind,
action: ApplicationHeartbeatFailureAction,
) -> bool {
snapshot_update(snapshot, |current| {
increment(&mut current.failures);
current.last_error = Some(kind);
current.state = ApplicationHeartbeatState::Running;
});
action == ApplicationHeartbeatFailureAction::Stop
}
fn increment(value: &mut u64) {
if let Some(next) = value.checked_add(1) {
*value = next;
}
}
fn snapshot_copy(
snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
) -> ApplicationHeartbeatSnapshot {
*lock_snapshot(snapshot)
}
fn snapshot_update<F>(snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>, update: F)
where
F: FnOnce(&mut ApplicationHeartbeatSnapshot),
{
let mut current = lock_snapshot(snapshot);
update(&mut current);
}
fn lock_snapshot<'a>(
snapshot: &'a Arc<Mutex<ApplicationHeartbeatSnapshot>>,
) -> MutexGuard<'a, ApplicationHeartbeatSnapshot> {
match snapshot.lock() {
Ok(guard) => guard,
Err(poisoned) => {
crate::log_e!(crate::common::log::log_def::LogType::WSC;
"application_heartbeat_snapshot", "error", "lock_poisoned_recovered");
poisoned.into_inner()
}
}
}
#[cfg(test)]
#[path = "application_heartbeat_tests.rs"]
mod tests;