use crate::common::log::log_def::LogType;
use bytes::Bytes;
use std::sync::Mutex;
use std::time::Duration;
use tokio::time::Instant;
#[derive(Clone, Debug, Eq, PartialEq)]
pub(crate) enum HeartbeatTick {
SendProbe(Bytes),
Waiting,
TimedOut,
}
pub(crate) struct HeartbeatState {
generation: u64,
inner: Mutex<HeartbeatStateInner>,
}
struct HeartbeatStateInner {
next_sequence: u64,
outstanding: Option<OutstandingProbe>,
}
struct OutstandingProbe {
payload: Bytes,
pong_timeout: Duration,
phase: ProbePhase,
}
enum ProbePhase {
Sending,
AwaitingPong { sent_at: Instant },
}
impl HeartbeatState {
pub(crate) fn new(generation: u64) -> Self {
crate::log_t!(LogType::WSC; "new", "generation", generation);
Self {
generation,
inner: Mutex::new(HeartbeatStateInner {
next_sequence: 1,
outstanding: None,
}),
}
}
pub(crate) fn on_tick(&self, now: Instant, pong_timeout: Duration) -> HeartbeatTick {
crate::log_t!(LogType::WSC; "on_tick", "now|pong_timeout", format!("{:?}", now), format!("{:?}", pong_timeout));
let mut inner = self.lock_inner();
let tick = match inner.outstanding.as_ref() {
Some(OutstandingProbe {
phase: ProbePhase::Sending,
..
}) => HeartbeatTick::Waiting,
Some(OutstandingProbe {
pong_timeout,
phase: ProbePhase::AwaitingPong { sent_at },
..
}) if now.saturating_duration_since(*sent_at) >= *pong_timeout => {
HeartbeatTick::TimedOut
}
Some(_) => HeartbeatTick::Waiting,
None => {
let sequence = inner.next_sequence;
inner.next_sequence = inner.next_sequence.wrapping_add(1).max(1);
let payload = heartbeat_payload(self.generation, sequence);
inner.outstanding = Some(OutstandingProbe {
payload: payload.clone(),
pong_timeout,
phase: ProbePhase::Sending,
});
HeartbeatTick::SendProbe(payload)
}
};
drop(inner);
match &tick {
HeartbeatTick::SendProbe(payload) => {
crate::log_s!(LogType::WSC; "on_tick", "generation|state|payload_bytes", self.generation, "Sending", payload.len())
}
HeartbeatTick::Waiting => {
crate::log_s!(LogType::WSC; "on_tick", "generation|state", self.generation, "probe_outstanding")
}
HeartbeatTick::TimedOut => {
crate::log_e!(LogType::WSC; "on_tick", "generation|error", self.generation, "PongTimeout")
}
}
tick
}
pub(crate) fn start_recovery_probe(
&self,
now: Instant,
pong_timeout: Duration,
) -> Option<Bytes> {
let mut inner = self.lock_inner();
let can_start = match inner.outstanding.as_ref() {
None => true,
Some(probe) => match probe.phase {
ProbePhase::Sending => false,
ProbePhase::AwaitingPong { sent_at } => sent_at
.checked_add(probe.pong_timeout)
.is_some_and(|deadline| now >= deadline),
},
};
if !can_start {
return None;
}
let sequence = inner.next_sequence;
inner.next_sequence = inner.next_sequence.wrapping_add(1).max(1);
let payload = heartbeat_payload(self.generation, sequence);
inner.outstanding = Some(OutstandingProbe {
payload: payload.clone(),
pong_timeout,
phase: ProbePhase::Sending,
});
drop(inner);
crate::log_s!(LogType::WSC; "ws.heartbeat_state.start_recovery_probe", "now|pong_timeout|generation|payload_bytes", format!("{now:?}"), pong_timeout, self.generation, payload.len());
Some(payload)
}
pub(crate) fn has_outstanding_probe(&self) -> bool {
self.lock_inner().outstanding.is_some()
}
pub(crate) fn mark_sent(&self, payload: &[u8], sent_at: Instant) -> bool {
crate::log_t!(LogType::WSC; "mark_sent", "payload_bytes|sent_at", payload.len(), format!("{:?}", sent_at));
let mut inner = self.lock_inner();
let Some(outstanding) = inner.outstanding.as_mut() else {
crate::log_s!(LogType::WSC; "mark_sent", "generation|state", self.generation, "probe_already_cleared");
return false;
};
if outstanding.payload.as_ref() != payload
|| !matches!(outstanding.phase, ProbePhase::Sending)
{
crate::log_s!(LogType::WSC; "mark_sent", "generation|state", self.generation, "probe_or_phase_mismatch");
return false;
}
outstanding.phase = ProbePhase::AwaitingPong { sent_at };
drop(inner);
crate::log_s!(LogType::WSC; "mark_sent", "generation|state", self.generation, "AwaitingPong");
true
}
pub(crate) fn acknowledge_pong(&self, payload: &[u8]) -> bool {
crate::log_t!(LogType::WSC; "acknowledge_pong", "payload_bytes", payload.len());
let now = Instant::now();
let mut inner = self.lock_inner();
let matches = inner.outstanding.as_ref().is_some_and(|probe| {
probe.payload.as_ref() == payload
&& match probe.phase {
ProbePhase::Sending => true,
ProbePhase::AwaitingPong { sent_at } => sent_at
.checked_add(probe.pong_timeout)
.is_some_and(|deadline| now < deadline),
}
});
if matches {
inner.outstanding = None;
}
drop(inner);
crate::log_s!(LogType::WSC; "acknowledge_pong", "generation|accepted", self.generation, matches);
matches
}
pub(crate) fn abandon_probe(&self, payload: &[u8]) {
crate::log_t!(LogType::WSC; "abandon_probe", "payload_bytes", payload.len());
let mut inner = self.lock_inner();
if inner
.outstanding
.as_ref()
.is_some_and(|probe| probe.payload.as_ref() == payload)
{
inner.outstanding = None;
drop(inner);
crate::log_s!(LogType::WSC; "abandon_probe", "generation|state", self.generation, "probe_abandoned");
}
}
pub(crate) fn pong_deadline(&self) -> Option<Instant> {
crate::log_t!(LogType::WSC; "pong_deadline");
let inner = self.lock_inner();
inner
.outstanding
.as_ref()
.and_then(|probe| match probe.phase {
ProbePhase::Sending => None,
ProbePhase::AwaitingPong { sent_at } => {
match sent_at.checked_add(probe.pong_timeout) {
Some(deadline) => Some(deadline),
None => {
crate::log_e!(LogType::WSC; "pong_deadline", "error", "pong_deadline_overflow");
Some(sent_at)
}
}
}
})
}
pub(crate) fn is_timed_out(&self, now: Instant) -> bool {
crate::log_t!(LogType::WSC; "is_timed_out", "now", format!("{:?}", now));
let timed_out = self.pong_deadline().is_some_and(|deadline| now >= deadline);
if timed_out {
crate::log_e!(LogType::WSC; "is_timed_out", "generation|error", self.generation, "PongTimeout");
}
timed_out
}
fn lock_inner(&self) -> std::sync::MutexGuard<'_, HeartbeatStateInner> {
crate::log_t!(LogType::WSC; "lock_inner");
match self.inner.lock() {
Ok(guard) => guard,
Err(poisoned) => {
crate::log_e!(LogType::WSC; "lock_inner", "error", "lock_poisoned_recovered");
poisoned.into_inner()
}
}
}
}
fn heartbeat_payload(generation: u64, sequence: u64) -> Bytes {
crate::log_t!(LogType::WSC; "heartbeat_payload", "generation|sequence", generation, sequence);
let mut payload = [0_u8; 16];
payload[..8].copy_from_slice(&generation.to_be_bytes());
payload[8..].copy_from_slice(&sequence.to_be_bytes());
Bytes::copy_from_slice(&payload)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::module::ws_client::test_support::{
check, check_eq, check_ne, test_error, TestResult,
};
#[tokio::test(start_paused = true)]
async fn matched_pong_clears_only_the_current_probe() -> TestResult {
let heartbeat = HeartbeatState::new(9);
let now = Instant::now();
let HeartbeatTick::SendProbe(first) = heartbeat.on_tick(now, Duration::from_secs(5)) else {
return Err(test_error("first tick must create a probe"));
};
check!(!first.is_empty())?;
check!(heartbeat.mark_sent(first.as_ref(), now))?;
check!(!heartbeat.acknowledge_pong(b"not-the-probe"))?;
check_eq!(
heartbeat.on_tick(now + Duration::from_secs(1), Duration::from_secs(5)),
HeartbeatTick::Waiting
)?;
check!(heartbeat.acknowledge_pong(first.as_ref()))?;
let HeartbeatTick::SendProbe(second) =
heartbeat.on_tick(now + Duration::from_secs(2), Duration::from_secs(5))
else {
return Err(test_error("matched Pong must allow the next probe"));
};
check!(!second.is_empty())?;
check_ne!(first, second)?;
Ok(())
}
#[tokio::test(start_paused = true)]
async fn timeout_starts_only_after_ping_send_succeeds() -> TestResult {
let heartbeat = HeartbeatState::new(11);
let started = Instant::now();
let HeartbeatTick::SendProbe(payload) = heartbeat.on_tick(started, Duration::from_secs(5))
else {
return Err(test_error("first tick must create a probe"));
};
check_eq!(
heartbeat.on_tick(started + Duration::from_secs(30), Duration::from_secs(5),),
HeartbeatTick::Waiting,
"a blocked Ping write is governed by its write timeout, not Pong timeout"
)?;
let sent_at = started + Duration::from_secs(30);
check!(heartbeat.mark_sent(payload.as_ref(), sent_at))?;
check_eq!(
heartbeat.on_tick(
sent_at + Duration::from_millis(4_999),
Duration::from_secs(5),
),
HeartbeatTick::Waiting
)?;
check_eq!(
heartbeat.on_tick(sent_at + Duration::from_secs(5), Duration::from_secs(5),),
HeartbeatTick::TimedOut
)?;
check!(heartbeat.is_timed_out(sent_at + Duration::from_secs(5)))?;
Ok(())
}
#[tokio::test(start_paused = true)]
async fn matching_pong_at_or_after_deadline_cannot_revive_the_connection() -> TestResult {
let timeout = Duration::from_secs(5);
let heartbeat = HeartbeatState::new(13);
let sent_at = Instant::now();
let HeartbeatTick::SendProbe(payload) = heartbeat.on_tick(sent_at, timeout) else {
return Err(test_error("first tick must create a probe"));
};
check!(heartbeat.mark_sent(payload.as_ref(), sent_at))?;
tokio::time::advance(timeout).await;
check!(!heartbeat.acknowledge_pong(payload.as_ref()))?;
check!(heartbeat.is_timed_out(Instant::now()))?;
check_eq!(
heartbeat.on_tick(Instant::now(), timeout),
HeartbeatTick::TimedOut
)?;
Ok(())
}
#[tokio::test(start_paused = true)]
async fn stale_generation_and_sequence_probes_cannot_change_current_probe() -> TestResult {
let timeout = Duration::from_secs(5);
for stale_source in ["previous-generation", "previous-sequence"] {
let heartbeat = HeartbeatState::new(42);
let previous_generation = HeartbeatState::new(41);
let now = Instant::now();
let stale_state = if stale_source == "previous-generation" {
&previous_generation
} else {
&heartbeat
};
let HeartbeatTick::SendProbe(stale) = stale_state.on_tick(now, timeout) else {
return Err(test_error("stale fixture did not create a real probe"));
};
if stale_source == "previous-sequence" {
check!(heartbeat.acknowledge_pong(&stale))?;
}
let HeartbeatTick::SendProbe(current) = heartbeat.on_tick(now, timeout) else {
return Err(test_error("current fixture did not create a real probe"));
};
check!(
!heartbeat.mark_sent(&stale, now),
"{stale_source}: stale probe marked the current Sending probe sent"
)?;
check_eq!(heartbeat.pong_deadline(), None)?;
check!(
!heartbeat.acknowledge_pong(&stale),
"{stale_source}: stale Pong cleared the current Sending probe"
)?;
check_eq!(heartbeat.on_tick(now, timeout), HeartbeatTick::Waiting)?;
check!(heartbeat.mark_sent(¤t, now))?;
let deadline = now
.checked_add(timeout)
.ok_or_else(|| test_error("bounded probe deadline was not representable"))?;
check_eq!(heartbeat.pong_deadline(), Some(deadline))?;
check!(
!heartbeat.acknowledge_pong(&stale),
"{stale_source}: stale Pong cleared the current AwaitingPong probe"
)?;
check_eq!(heartbeat.pong_deadline(), Some(deadline))?;
heartbeat.abandon_probe(&stale);
check_eq!(
heartbeat.pong_deadline(),
Some(deadline),
"{stale_source}: stale failure abandoned the current probe"
)?;
tokio::time::advance(Duration::from_secs(4)).await;
check!(!heartbeat.is_timed_out(Instant::now()))?;
check!(!heartbeat.mark_sent(&stale, Instant::now()))?;
check!(!heartbeat.acknowledge_pong(&stale))?;
check_eq!(heartbeat.pong_deadline(), Some(deadline))?;
tokio::time::advance(Duration::from_secs(1)).await;
check!(heartbeat.is_timed_out(Instant::now()))?;
check!(!heartbeat.acknowledge_pong(¤t))?;
check_eq!(
heartbeat.on_tick(Instant::now(), timeout),
HeartbeatTick::TimedOut
)?;
}
Ok(())
}
#[tokio::test(start_paused = true)]
async fn abandoned_probe_allows_a_new_normal_probe() -> TestResult {
let heartbeat = HeartbeatState::new(79);
let now = Instant::now();
let HeartbeatTick::SendProbe(first) = heartbeat.on_tick(now, Duration::from_secs(5)) else {
return Err(test_error("normal probe was not created"));
};
heartbeat.abandon_probe(first.as_ref());
let HeartbeatTick::SendProbe(next) =
heartbeat.on_tick(now + Duration::from_millis(1), Duration::from_secs(5))
else {
return Err(test_error("abandoning a probe must allow a new probe"));
};
check_ne!(first, next)?;
check!(heartbeat.mark_sent(next.as_ref(), now + Duration::from_millis(1)))?;
check!(!heartbeat.acknowledge_pong(first.as_ref()))?;
check!(heartbeat.acknowledge_pong(next.as_ref()))?;
Ok(())
}
#[tokio::test(start_paused = true)]
async fn recovery_probe_replaces_only_expired_probe() -> TestResult {
let heartbeat = HeartbeatState::new(80);
let timeout = Duration::from_secs(5);
let now = Instant::now();
let HeartbeatTick::SendProbe(first) = heartbeat.on_tick(now, timeout) else {
return Err(test_error("normal probe was not created"));
};
check!(heartbeat.start_recovery_probe(now, timeout).is_none())?;
check!(heartbeat.mark_sent(first.as_ref(), now))?;
check!(heartbeat
.start_recovery_probe(now + timeout - Duration::from_nanos(1), timeout)
.is_none())?;
let expired = now
.checked_add(timeout)
.ok_or_else(|| test_error("test deadline overflow"))?;
let recovery = heartbeat
.start_recovery_probe(expired, timeout)
.ok_or_else(|| test_error("expired probe did not start recovery"))?;
check_ne!(first, recovery)?;
check!(!heartbeat.acknowledge_pong(first.as_ref()))?;
check!(heartbeat.has_outstanding_probe())?;
check!(heartbeat.mark_sent(recovery.as_ref(), expired))?;
check!(heartbeat.acknowledge_pong(recovery.as_ref()))?;
check!(!heartbeat.has_outstanding_probe())?;
Ok(())
}
#[tokio::test(start_paused = true)]
async fn recovery_probe_starts_from_idle_state() -> TestResult {
let heartbeat = HeartbeatState::new(81);
let payload = heartbeat
.start_recovery_probe(Instant::now(), Duration::from_secs(1))
.ok_or_else(|| test_error("idle state did not start recovery"))?;
check!(heartbeat.has_outstanding_probe())?;
check!(heartbeat.mark_sent(payload.as_ref(), Instant::now()))?;
check!(heartbeat.acknowledge_pong(payload.as_ref()))?;
Ok(())
}
}