use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Duration;
use tokio::sync::Notify;
use tokio::time::Instant;
pub(super) const IDLE_INTERVAL: Duration = Duration::from_secs(20);
const SILENT_STRIKES_BEFORE_RESTART: u8 = 2;
#[derive(Default)]
pub(super) struct ChannelActivity {
seq: AtomicU64,
changed: Notify,
}
impl ChannelActivity {
pub(super) fn record(&self) {
self.seq.fetch_add(1, Ordering::Release);
self.changed.notify_one();
}
pub(super) fn seq(&self) -> u64 {
self.seq.load(Ordering::Acquire)
}
pub(super) async fn changed_after(&self, observed: u64) -> u64 {
loop {
let notified = self.changed.notified();
tokio::pin!(notified);
let _ = notified.as_mut().enable();
let current = self.seq();
if current != observed {
return current;
}
notified.await;
}
}
}
#[derive(Clone, Copy, PartialEq, Eq)]
pub(super) enum PingOutcome {
Delivered,
AllSilent,
ChannelFailed,
}
#[derive(Clone, Copy, PartialEq, Eq)]
pub(super) enum LivenessDecision {
Continue,
Restart,
}
pub(super) struct CaptureLiveness {
activity_seq: u64,
idle_deadline: Instant,
silent_strikes: u8,
}
impl CaptureLiveness {
pub(super) fn new(now: Instant, activity_seq: u64) -> Self {
Self {
activity_seq,
idle_deadline: now + IDLE_INTERVAL,
silent_strikes: 0,
}
}
pub(super) fn activity_seq(&self) -> u64 {
self.activity_seq
}
pub(super) fn idle_deadline(&self) -> Instant {
self.idle_deadline
}
pub(super) fn record_activity(&mut self, now: Instant, seq: u64) {
let _ = self.take_activity(now, seq);
}
pub(super) fn ping_due(&mut self, now: Instant, seq: u64) -> bool {
!self.take_activity(now, seq) && now >= self.idle_deadline
}
pub(super) fn finish_ping(
&mut self,
now: Instant,
seq: u64,
outcome: PingOutcome,
) -> LivenessDecision {
let activity = self.take_activity(now, seq);
self.idle_deadline = now + IDLE_INTERVAL;
if outcome == PingOutcome::ChannelFailed {
return LivenessDecision::Restart;
}
if activity || outcome == PingOutcome::Delivered {
self.silent_strikes = 0;
return LivenessDecision::Continue;
}
self.silent_strikes = self.silent_strikes.saturating_add(1);
if self.silent_strikes >= SILENT_STRIKES_BEFORE_RESTART {
LivenessDecision::Restart
} else {
LivenessDecision::Continue
}
}
fn take_activity(&mut self, now: Instant, seq: u64) -> bool {
if seq == self.activity_seq {
return false;
}
self.activity_seq = seq;
self.idle_deadline = now + IDLE_INTERVAL;
self.silent_strikes = 0;
true
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test(start_paused = true)]
async fn active_traffic_defers_the_idle_ping() {
let activity = ChannelActivity::default();
let mut liveness = CaptureLiveness::new(Instant::now(), activity.seq());
let almost_idle = IDLE_INTERVAL
.checked_sub(Duration::from_secs(1))
.expect("the test offset is shorter than the idle interval");
tokio::time::advance(almost_idle).await;
activity.record();
liveness.record_activity(Instant::now(), activity.seq());
tokio::time::advance(Duration::from_secs(1)).await;
assert!(!liveness.ping_due(Instant::now(), activity.seq()));
tokio::time::advance(almost_idle).await;
assert!(liveness.ping_due(Instant::now(), activity.seq()));
}
#[tokio::test(start_paused = true)]
async fn idle_ping_is_scheduled_after_each_full_interval() {
let activity = ChannelActivity::default();
let mut liveness = CaptureLiveness::new(Instant::now(), activity.seq());
let almost_idle = IDLE_INTERVAL
.checked_sub(Duration::from_millis(1))
.expect("the test offset is shorter than the idle interval");
tokio::time::advance(almost_idle).await;
assert!(!liveness.ping_due(Instant::now(), activity.seq()));
tokio::time::advance(Duration::from_millis(1)).await;
assert!(liveness.ping_due(Instant::now(), activity.seq()));
assert!(matches!(
liveness.finish_ping(Instant::now(), activity.seq(), PingOutcome::AllSilent,),
LivenessDecision::Continue
));
tokio::time::advance(almost_idle).await;
assert!(!liveness.ping_due(Instant::now(), activity.seq()));
tokio::time::advance(Duration::from_millis(1)).await;
assert!(liveness.ping_due(Instant::now(), activity.seq()));
}
#[tokio::test(start_paused = true)]
async fn delivery_resets_a_silent_strike() {
let activity = ChannelActivity::default();
let mut liveness = CaptureLiveness::new(Instant::now(), activity.seq());
tokio::time::advance(IDLE_INTERVAL).await;
assert!(matches!(
liveness.finish_ping(Instant::now(), activity.seq(), PingOutcome::AllSilent,),
LivenessDecision::Continue
));
tokio::time::advance(IDLE_INTERVAL).await;
assert!(matches!(
liveness.finish_ping(Instant::now(), activity.seq(), PingOutcome::Delivered,),
LivenessDecision::Continue
));
tokio::time::advance(IDLE_INTERVAL).await;
assert!(matches!(
liveness.finish_ping(Instant::now(), activity.seq(), PingOutcome::AllSilent,),
LivenessDecision::Continue
));
tokio::time::advance(IDLE_INTERVAL).await;
assert!(matches!(
liveness.finish_ping(Instant::now(), activity.seq(), PingOutcome::AllSilent,),
LivenessDecision::Restart
));
}
#[tokio::test(start_paused = true)]
async fn channel_failure_restarts_without_a_second_probe() {
let activity = ChannelActivity::default();
let mut liveness = CaptureLiveness::new(Instant::now(), activity.seq());
tokio::time::advance(IDLE_INTERVAL).await;
assert!(matches!(
liveness.finish_ping(Instant::now(), activity.seq(), PingOutcome::ChannelFailed,),
LivenessDecision::Restart
));
}
#[tokio::test(start_paused = true)]
async fn activity_before_wait_registration_is_not_lost() {
let activity = ChannelActivity::default();
activity.record();
let seq = tokio::time::timeout(Duration::from_secs(1), activity.changed_after(0))
.await
.expect("pre-existing activity must make the waiter ready");
assert_eq!(seq, activity.seq());
}
}