use std::time::Duration;
use tokio_util::sync::CancellationToken;
pub(crate) const INTERVAL: Duration = Duration::from_secs(4);
pub(crate) const TIMEOUT: Duration = Duration::from_secs(5);
pub(crate) async fn run<F, Fut>(mut beat: F, stop: CancellationToken)
where
F: FnMut() -> Fut,
Fut: Future<Output = Result<(), tonic::Status>>,
{
loop {
tokio::select! {
() = stop.cancelled() => return,
result = beat() => {
if let Err(status) = result {
tracing::warn!(
code = ?status.code(),
message = status.message(),
"worker heartbeat failed",
);
}
}
}
tokio::select! {
() = stop.cancelled() => return,
() = tokio::time::sleep(INTERVAL) => {}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
use tokio::time::Instant;
use tokio_util::sync::CancellationToken;
use tonic::Status;
#[tokio::test(start_paused = true)]
async fn beats_immediately_and_then_every_four_seconds() {
let start = Instant::now();
let beats: Arc<Mutex<Vec<Duration>>> = Arc::new(Mutex::new(Vec::new()));
let stop = CancellationToken::new();
let handle = tokio::spawn({
let beats = Arc::clone(&beats);
let stop = stop.clone();
async move {
run(
move || {
let beats = Arc::clone(&beats);
async move {
beats.lock().unwrap().push(start.elapsed());
Ok(())
}
},
stop,
)
.await;
}
});
tokio::time::sleep(Duration::from_secs(9)).await;
stop.cancel();
handle.await.unwrap();
let recorded = beats.lock().unwrap().clone();
assert_eq!(
recorded,
[
Duration::ZERO,
Duration::from_secs(4),
Duration::from_secs(8)
]
);
}
#[tokio::test(start_paused = true)]
async fn a_slow_beat_pushes_the_next_one_back() {
let start = Instant::now();
let starts: Arc<Mutex<Vec<Duration>>> = Arc::new(Mutex::new(Vec::new()));
let stop = CancellationToken::new();
let handle = tokio::spawn({
let starts = Arc::clone(&starts);
let stop = stop.clone();
async move {
run(
move || {
let starts = Arc::clone(&starts);
async move {
starts.lock().unwrap().push(start.elapsed());
tokio::time::sleep(Duration::from_secs(1)).await;
Ok(())
}
},
stop,
)
.await;
}
});
tokio::time::sleep(Duration::from_secs(12)).await;
stop.cancel();
handle.await.unwrap();
let recorded = starts.lock().unwrap().clone();
assert_eq!(
recorded,
[
Duration::ZERO,
Duration::from_secs(5),
Duration::from_secs(10)
],
"a one-second beat plus a four-second wait is five seconds apart",
);
}
#[tokio::test(start_paused = true)]
async fn a_failed_beat_does_not_stop_the_loop() {
let calls = Arc::new(AtomicUsize::new(0));
let stop = CancellationToken::new();
let handle = tokio::spawn({
let calls = Arc::clone(&calls);
let stop = stop.clone();
async move {
run(
move || {
let calls = Arc::clone(&calls);
async move {
if calls.fetch_add(1, Ordering::SeqCst) == 0 {
Err(Status::unavailable("engine restarting"))
} else {
Ok(())
}
}
},
stop,
)
.await;
}
});
tokio::time::sleep(Duration::from_secs(9)).await;
stop.cancel();
handle.await.unwrap();
assert_eq!(calls.load(Ordering::SeqCst), 3);
}
#[tokio::test(start_paused = true)]
async fn stops_when_cancelled() {
let calls = Arc::new(AtomicUsize::new(0));
let stop = CancellationToken::new();
let handle = tokio::spawn({
let calls = Arc::clone(&calls);
let stop = stop.clone();
async move {
run(
move || {
let calls = Arc::clone(&calls);
async move {
calls.fetch_add(1, Ordering::SeqCst);
Ok(())
}
},
stop,
)
.await;
}
});
tokio::time::sleep(Duration::from_secs(1)).await;
stop.cancel();
handle.await.unwrap();
assert_eq!(calls.load(Ordering::SeqCst), 1);
tokio::time::sleep(Duration::from_secs(10)).await;
assert_eq!(calls.load(Ordering::SeqCst), 1);
}
}