use super::{
DisconnectedPolicy, IncomingMessage, Message, MessageReceiver, ReceiveOptions, ReceiveOverflow,
SendOptions, Sender,
};
use crate::error::{ErrorKind, ErrorStage, ReceiveError};
use crate::module::ws_client::session_runtime::SessionRuntime;
use crate::{NetError, Result};
use std::panic::{catch_unwind, AssertUnwindSafe};
use std::sync::{Arc, Mutex, MutexGuard};
use std::time::{Duration, Instant as StdInstant, SystemTime};
use tokio::task::JoinHandle;
use tokio::time::{self, Instant};
use tokio_util::sync::CancellationToken;
pub type ApplicationHeartbeatMatcher =
Arc<dyn Fn(&IncomingMessage) -> bool + Send + Sync + 'static>;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ApplicationHeartbeatFailureAction {
Continue,
Stop,
}
#[derive(Clone)]
pub struct ApplicationHeartbeatConfig {
pub interval: Duration,
pub payload: Message,
pub send_timeout: Duration,
pub response_timeout: Option<Duration>,
pub max_missed_responses: u32,
pub matcher: Option<ApplicationHeartbeatMatcher>,
pub receive_options: ReceiveOptions,
pub failure_action: ApplicationHeartbeatFailureAction,
}
impl ApplicationHeartbeatConfig {
pub fn new(interval: Duration, payload: impl Into<Message>) -> Self {
Self {
interval,
payload: payload.into(),
send_timeout: Duration::from_secs(10),
response_timeout: None,
max_missed_responses: 1,
matcher: None,
receive_options: ReceiveOptions {
overflow: ReceiveOverflow::DropOldest,
..ReceiveOptions::default()
},
failure_action: ApplicationHeartbeatFailureAction::Continue,
}
}
pub fn with_send_timeout(mut self, timeout: Duration) -> Self {
self.send_timeout = timeout;
self
}
pub fn with_response_matcher(
mut self,
timeout: Duration,
matcher: impl Fn(&IncomingMessage) -> bool + Send + Sync + 'static,
) -> Self {
self.response_timeout = Some(timeout);
self.matcher = Some(Arc::new(matcher));
self
}
pub fn with_max_missed_responses(mut self, max_missed_responses: u32) -> Self {
self.max_missed_responses = max_missed_responses;
self
}
pub fn with_receive_options(mut self, receive_options: ReceiveOptions) -> Self {
self.receive_options = receive_options;
self
}
pub fn with_failure_action(
mut self,
failure_action: ApplicationHeartbeatFailureAction,
) -> Self {
self.failure_action = failure_action;
self
}
pub fn validate(&self) -> Result<()> {
if self.interval.is_zero() {
return Err(NetError::config(
"application_heartbeat.interval",
"must be greater than zero",
));
}
if Instant::now().checked_add(self.interval).is_none() {
return Err(NetError::config(
"application_heartbeat.interval",
"cannot be represented as a timer deadline",
));
}
if self.send_timeout.is_zero() {
return Err(NetError::config(
"application_heartbeat.send_timeout",
"must be greater than zero",
));
}
if StdInstant::now().checked_add(self.send_timeout).is_none() {
return Err(NetError::config(
"application_heartbeat.send_timeout",
"cannot be represented as a deadline",
));
}
if self.max_missed_responses == 0 {
return Err(NetError::config(
"application_heartbeat.max_missed_responses",
"must be greater than zero",
));
}
if self.matcher.is_some() && self.receive_options.include_control_frames {
return Err(NetError::config(
"application_heartbeat.receive_options.include_control_frames",
"must be false so control Pong frames cannot acknowledge application heartbeats",
));
}
if self.matcher.is_some() && self.response_timeout.is_none() {
return Err(NetError::config(
"application_heartbeat.response_timeout",
"is required when matcher is set",
));
}
if let Some(timeout) = self.response_timeout {
if StdInstant::now().checked_add(timeout).is_none() {
return Err(NetError::config(
"application_heartbeat.response_timeout",
"must fit an absolute deadline",
));
}
if timeout.is_zero() {
return Err(NetError::config(
"application_heartbeat.response_timeout",
"must be greater than zero",
));
}
if Instant::now().checked_add(timeout).is_none() {
return Err(NetError::config(
"application_heartbeat.response_timeout",
"cannot be represented as a deadline",
));
}
if self.matcher.is_none() {
return Err(NetError::config(
"application_heartbeat.matcher",
"is required when response_timeout is set",
));
}
}
if self.matcher.is_some() {
self.receive_options.validate()?;
}
Ok(())
}
}
impl std::fmt::Debug for ApplicationHeartbeatConfig {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
.debug_struct("ApplicationHeartbeatConfig")
.field("interval", &self.interval)
.field("payload_bytes", &self.payload.len())
.field("send_timeout", &self.send_timeout)
.field("response_timeout", &self.response_timeout)
.field("max_missed_responses", &self.max_missed_responses)
.field("has_matcher", &self.matcher.is_some())
.field("receive_options", &self.receive_options)
.field("failure_action", &self.failure_action)
.finish()
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ApplicationHeartbeatState {
Running,
WaitingForResponse,
Stopped,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct ApplicationHeartbeatSnapshot {
pub state: ApplicationHeartbeatState,
pub sent: u64,
pub matched_responses: u64,
pub missed_responses: u64,
pub failures: u64,
pub last_error: Option<ErrorKind>,
}
impl Default for ApplicationHeartbeatSnapshot {
fn default() -> Self {
Self {
state: ApplicationHeartbeatState::Running,
sent: 0,
matched_responses: 0,
missed_responses: 0,
failures: 0,
last_error: None,
}
}
}
pub struct ApplicationHeartbeatHandle {
cancellation: CancellationToken,
snapshot: Arc<Mutex<ApplicationHeartbeatSnapshot>>,
task: Option<JoinHandle<()>>,
}
impl ApplicationHeartbeatHandle {
pub(crate) fn new(
sender: Sender,
receiver: Option<MessageReceiver>,
config: ApplicationHeartbeatConfig,
) -> Result<Self> {
config.validate()?;
let cancellation = CancellationToken::new();
let snapshot = Arc::new(Mutex::new(ApplicationHeartbeatSnapshot::default()));
let runtime = sender.runtime.clone();
let task = runtime.clone().spawn(run_heartbeat(
sender,
receiver,
config,
cancellation.clone(),
snapshot.clone(),
runtime,
))?;
Ok(Self {
cancellation,
snapshot,
task: Some(task),
})
}
pub fn stop(&self) {
self.cancellation.cancel();
}
pub fn snapshot(&self) -> ApplicationHeartbeatSnapshot {
snapshot_copy(&self.snapshot)
}
pub async fn stopped(mut self) -> Result<ApplicationHeartbeatSnapshot> {
let task = self
.task
.take()
.ok_or_else(|| NetError::from(ErrorKind::Internal).with_stage(ErrorStage::Runtime))?;
task.await.map_err(|error| {
NetError::with_source(ErrorKind::EngineDropped, error).with_stage(ErrorStage::Runtime)
})?;
Ok(snapshot_copy(&self.snapshot))
}
}
impl Drop for ApplicationHeartbeatHandle {
fn drop(&mut self) {
self.cancellation.cancel();
if let Some(task) = self.task.take() {
task.abort();
}
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum ResponseWait {
Matched,
TimedOut,
Closed,
Cancelled,
Failed(HeartbeatFailure),
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum HeartbeatPhase {
SendOptions,
Send,
Connection,
ResponseWait,
Matcher,
Receive,
}
impl HeartbeatPhase {
fn name(self) -> &'static str {
match self {
Self::SendOptions => "send_options",
Self::Send => "send",
Self::Connection => "connection",
Self::ResponseWait => "response_wait",
Self::Matcher => "matcher",
Self::Receive => "receive",
}
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct HeartbeatFailure {
phase: HeartbeatPhase,
kind: ErrorKind,
stage: Option<ErrorStage>,
}
const HEARTBEAT_STAGE_SLOTS: usize = ErrorStage::Runtime as usize + 2;
#[derive(Default)]
struct HeartbeatDiagnostics {
client_id: u64,
last: Option<HeartbeatFailure>,
failures: u64,
reported: [[u64; HEARTBEAT_STAGE_SLOTS]; 6],
}
impl HeartbeatDiagnostics {
fn should_report(&mut self, failure: HeartbeatFailure, enabled: bool) -> bool {
if matches!(failure.kind, ErrorKind::Cancelled | ErrorKind::Closed) {
self.last = None;
self.failures = 0;
self.reported = [[0; HEARTBEAT_STAGE_SLOTS]; 6];
return false;
}
self.last = Some(failure);
self.failures = self.failures.saturating_add(1);
if !enabled {
self.reported = [[0; HEARTBEAT_STAGE_SLOTS]; 6];
return false;
}
let bit = 1_u64
.checked_shl(failure.kind as u32)
.unwrap_or(1_u64 << 63);
let stage = failure
.stage
.map_or(0, |stage| stage as usize + 1)
.min(HEARTBEAT_STAGE_SLOTS - 1);
let reported = &mut self.reported[failure.phase as usize][stage];
let first = *reported & bit == 0;
*reported |= bit;
first
}
fn failed(&mut self, session_id: u64, failure: HeartbeatFailure) {
if self.should_report(
failure,
crate::OpenNetLogger::is_enabled_for(crate::LogType::WSC, crate::LogLevel::Error),
) {
crate::log_e!(crate::LogType::WSC; "ws.heartbeat.failed",
"client_id|session_id|phase|kind|stage|failures", self.client_id, session_id, failure.phase.name(),
format!("{:?}", failure.kind), failure.stage.map(|stage| format!("{stage:?}")), self.failures);
}
}
fn error(&mut self, session_id: u64, phase: HeartbeatPhase, error: &NetError) {
self.failed(
session_id,
HeartbeatFailure {
phase,
kind: error.kind(),
stage: error.context().stage,
},
);
}
fn recovered(&mut self, session_id: u64) {
if let Some(failure) = self.last.take() {
crate::log_s!(crate::LogType::WSC, crate::LogLevel::Key; "ws.heartbeat.recovered",
"client_id|session_id|phase|failures", self.client_id, session_id, failure.phase.name(), self.failures);
}
self.failures = 0;
self.reported = [[0; HEARTBEAT_STAGE_SLOTS]; 6];
}
fn stopped(&self, session_id: u64) {
if let Some(failure) = self.last {
crate::log_s!(crate::LogType::WSC, crate::LogLevel::Key; "ws.heartbeat.stopped",
"client_id|session_id|phase|kind|failures", self.client_id, session_id, failure.phase.name(), format!("{:?}", failure.kind), self.failures);
}
}
}
async fn run_heartbeat(
sender: Sender,
mut receiver: Option<MessageReceiver>,
config: ApplicationHeartbeatConfig,
cancellation: CancellationToken,
snapshot: Arc<Mutex<ApplicationHeartbeatSnapshot>>,
runtime: Arc<SessionRuntime>,
) {
let now = Instant::now();
let first_tick = match now.checked_add(config.interval) {
Some(deadline) => deadline,
None => now,
};
let mut ticker = time::interval_at(first_tick, config.interval);
ticker.set_missed_tick_behavior(time::MissedTickBehavior::Skip);
let mut consecutive_missed = 0_u32;
let mut stopped_by_policy = false;
let session_closed = runtime.closed_token();
let session_id = runtime.id.as_u64();
let mut diagnostics = HeartbeatDiagnostics {
client_id: runtime.client_id.as_u64(),
..Default::default()
};
'heartbeat: loop {
tokio::select! {
_ = cancellation.cancelled() => break 'heartbeat,
_ = session_closed.cancelled() => break 'heartbeat,
_ = ticker.tick() => {
snapshot_update(&snapshot, |current| {
current.state = ApplicationHeartbeatState::Running;
});
let options = match send_options(config.send_timeout) {
Ok(options) => options,
Err(error) => {
diagnostics.error(session_id, HeartbeatPhase::SendOptions, &error);
consecutive_missed = 0;
let should_stop = record_failure(
&snapshot,
&error,
config.failure_action,
);
if should_stop {
stopped_by_policy = true;
break 'heartbeat;
}
continue;
}
};
let sent_at = SystemTime::now();
let send = sender
.message(config.payload.clone())
.options(options)
.send();
let send_result = tokio::select! {
_ = cancellation.cancelled() => break 'heartbeat,
_ = session_closed.cancelled() => break 'heartbeat,
result = send => result,
};
if let Err(error) = send_result {
diagnostics.error(session_id, HeartbeatPhase::Send, &error);
consecutive_missed = 0;
let should_stop = record_failure(
&snapshot,
&error,
config.failure_action,
);
if should_stop {
stopped_by_policy = true;
break 'heartbeat;
}
continue;
}
snapshot_update(&snapshot, |current| {
increment(&mut current.sent);
});
let (Some(matcher), Some(receiver)) =
(config.matcher.as_ref(), receiver.as_mut())
else {
diagnostics.recovered(session_id);
continue;
};
let connection_id = match runtime.current_connection_id() {
Ok(Some(connection_id)) => connection_id,
Ok(None) => {
consecutive_missed = 0;
let error = NetError::from(ErrorKind::NotConnected)
.with_stage(ErrorStage::Runtime);
let should_stop = record_failure(
&snapshot,
&error,
config.failure_action,
);
if should_stop {
diagnostics.last = Some(HeartbeatFailure {
phase: HeartbeatPhase::Connection,
kind: ErrorKind::NotConnected,
stage: Some(ErrorStage::Runtime),
});
diagnostics.failures = diagnostics.failures.saturating_add(1);
stopped_by_policy = true;
break 'heartbeat;
}
continue;
}
Err(error) => {
diagnostics.error(session_id, HeartbeatPhase::Connection, &error);
consecutive_missed = 0;
let should_stop = record_failure(
&snapshot,
&error,
config.failure_action,
);
if should_stop {
stopped_by_policy = true;
break 'heartbeat;
}
continue;
}
};
snapshot_update(&snapshot, |current| {
current.state = ApplicationHeartbeatState::WaitingForResponse;
});
let timeout = match config.response_timeout {
Some(timeout) => timeout,
None => continue,
};
match wait_for_response(
receiver,
matcher,
timeout,
&cancellation,
&session_closed,
Some(connection_id),
sent_at,
).await {
ResponseWait::Matched => {
consecutive_missed = 0;
snapshot_update(&snapshot, |current| {
increment(&mut current.matched_responses);
current.state = ApplicationHeartbeatState::Running;
});
diagnostics.recovered(session_id);
}
ResponseWait::TimedOut => {
diagnostics.failed(session_id, HeartbeatFailure {
phase: HeartbeatPhase::ResponseWait, kind: ErrorKind::TimedOut, stage: Some(ErrorStage::Receive),
});
consecutive_missed = consecutive_missed.saturating_add(1);
snapshot_update(&snapshot, |current| {
increment(&mut current.missed_responses);
current.last_error = Some(ErrorKind::TimedOut);
increment(&mut current.failures);
current.state = ApplicationHeartbeatState::Running;
});
if config.failure_action == ApplicationHeartbeatFailureAction::Stop
&& consecutive_missed >= config.max_missed_responses
{
stopped_by_policy = true;
break 'heartbeat;
}
}
ResponseWait::Closed | ResponseWait::Cancelled => break 'heartbeat,
ResponseWait::Failed(failure) => {
diagnostics.failed(session_id, failure);
consecutive_missed = 0;
let should_stop = record_failure_kind(
&snapshot,
failure.kind,
config.failure_action,
);
if should_stop {
stopped_by_policy = true;
break 'heartbeat;
}
}
}
}
}
}
if let Some(receiver) = receiver.as_ref() {
receiver.unsubscribe();
}
snapshot_update(&snapshot, |current| {
current.state = ApplicationHeartbeatState::Stopped;
});
if stopped_by_policy {
diagnostics.stopped(session_id);
}
}
async fn wait_for_response(
receiver: &mut MessageReceiver,
matcher: &ApplicationHeartbeatMatcher,
timeout: Duration,
cancellation: &CancellationToken,
session_closed: &CancellationToken,
connection_id: Option<crate::ws::ConnectionId>,
sent_at: SystemTime,
) -> ResponseWait {
let deadline = time::sleep(timeout);
tokio::pin!(deadline);
loop {
tokio::select! {
_ = cancellation.cancelled() => return ResponseWait::Cancelled,
_ = session_closed.cancelled() => return ResponseWait::Closed,
_ = &mut deadline => return ResponseWait::TimedOut,
result = receiver.recv() => {
match result {
Ok(Some(message)) => {
if connection_id != Some(message.connection_id()) {
continue;
}
if message.received_at().duration_since(sent_at).is_err() {
continue;
}
let matches = match catch_unwind(AssertUnwindSafe(|| matcher(&message))) {
Ok(matches) => matches,
Err(_) => return ResponseWait::Failed(HeartbeatFailure {
phase: HeartbeatPhase::Matcher, kind: ErrorKind::CallbackPanicked, stage: Some(ErrorStage::Receive),
}),
};
if matches {
return ResponseWait::Matched;
}
}
Ok(None) => return ResponseWait::Closed,
Err(ReceiveError::Lagged { .. }) => continue,
Err(ReceiveError::Failed(error)) => {
return ResponseWait::Failed(HeartbeatFailure {
phase: HeartbeatPhase::Receive, kind: error.kind(), stage: error.context().stage,
});
}
}
}
}
}
}
fn send_options(timeout: Duration) -> Result<SendOptions> {
let deadline = StdInstant::now().checked_add(timeout).ok_or_else(|| {
NetError::config(
"application_heartbeat.send_timeout",
"cannot be represented as a deadline",
)
})?;
Ok(SendOptions {
disconnected: DisconnectedPolicy::WaitForReconnect,
enqueue_timeout: Some(timeout),
write_timeout: timeout,
deadline: Some(deadline),
..SendOptions::default()
})
}
fn record_failure(
snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
error: &NetError,
action: ApplicationHeartbeatFailureAction,
) -> bool {
record_failure_kind(snapshot, error.kind(), action)
}
fn record_failure_kind(
snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
kind: ErrorKind,
action: ApplicationHeartbeatFailureAction,
) -> bool {
snapshot_update(snapshot, |current| {
increment(&mut current.failures);
current.last_error = Some(kind);
current.state = ApplicationHeartbeatState::Running;
});
action == ApplicationHeartbeatFailureAction::Stop
}
fn increment(value: &mut u64) {
if let Some(next) = value.checked_add(1) {
*value = next;
}
}
fn snapshot_copy(
snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>,
) -> ApplicationHeartbeatSnapshot {
*lock_snapshot(snapshot)
}
fn snapshot_update<F>(snapshot: &Arc<Mutex<ApplicationHeartbeatSnapshot>>, update: F)
where
F: FnOnce(&mut ApplicationHeartbeatSnapshot),
{
let mut current = lock_snapshot(snapshot);
update(&mut current);
}
fn lock_snapshot<'a>(
snapshot: &'a Arc<Mutex<ApplicationHeartbeatSnapshot>>,
) -> MutexGuard<'a, ApplicationHeartbeatSnapshot> {
match snapshot.lock() {
Ok(guard) => guard,
Err(poisoned) => {
crate::log_e!(crate::common::log::log_def::LogType::WSC;
"ws.heartbeat.lock_snapshot", "arg_snapshot_present|error", true, "lock_poisoned_recovered");
poisoned.into_inner()
}
}
}
#[cfg(test)]
#[path = "application_heartbeat_tests.rs"]
mod tests;