pub(crate) mod os;
pub(crate) mod ready;
use core::future::Future;
use core::pin::Pin;
use core::time::Duration;
use std::sync::Arc;
use tokio::sync::mpsc;
use shep_core::config::{ProbeConfig, ProbeTarget};
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ProbeFailure {
Timeout,
Transport(String),
Rejected(String),
}
pub trait Prober: Send + Sync + 'static {
fn probe<'a>(
&'a self,
target: &'a ProbeTarget,
timeout: Duration,
) -> Pin<Box<dyn Future<Output = Result<(), ProbeFailure>> + Send + 'a>>;
}
const MIN_PROBE_INTERVAL: Duration = Duration::from_millis(1_000);
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct LivenessFailure {
pub id: u32,
pub pid: u32,
}
pub(crate) fn spawn_liveness_task(
id: u32,
pid: u32,
config: ProbeConfig,
target: ProbeTarget,
prober: Arc<dyn Prober>,
failures: mpsc::Sender<LivenessFailure>,
) -> tokio::task::JoinHandle<()> {
let interval = config.interval.as_duration().max(MIN_PROBE_INTERVAL);
let timeout = config.timeout.as_duration();
let threshold = config.failure_threshold;
tokio::spawn(async move {
let mut consecutive_failures: u32 = 0;
loop {
tokio::time::sleep(interval).await;
match prober.probe(&target, timeout).await {
Ok(()) => consecutive_failures = 0,
Err(_failure) => {
consecutive_failures += 1;
if consecutive_failures >= threshold {
let _ = failures.send(LivenessFailure { id, pid }).await;
return;
}
}
}
}
})
}
#[cfg(test)]
mod tests {
use core::time::Duration;
use std::sync::Arc;
use tokio::sync::mpsc;
use super::*;
use crate::testing::{ScriptedProber, probe_config};
use shep_core::config::ProbeKind;
const FAILURE_WAIT: Duration = Duration::from_secs(120);
fn target() -> ProbeTarget {
ProbeTarget::Tcp {
host: "localhost".to_string(),
port: 5432,
}
}
fn config_with_threshold(threshold: u32) -> ProbeConfig {
ProbeConfig {
failure_threshold: threshold,
..probe_config(ProbeKind::Tcp, "localhost:5432")
}
}
fn intervals(interval: Duration, n: u32) -> Duration {
interval * n + Duration::from_millis(1)
}
async fn spawn_and_settle(
id: u32,
pid: u32,
config: ProbeConfig,
prober: Arc<dyn Prober>,
failures: mpsc::Sender<LivenessFailure>,
) -> tokio::task::JoinHandle<()> {
let handle = spawn_liveness_task(id, pid, config, target(), prober, failures);
tokio::task::yield_now().await;
handle
}
async fn expect_failure(rx: &mut mpsc::Receiver<LivenessFailure>, id: u32) -> LivenessFailure {
match tokio::time::timeout(FAILURE_WAIT, rx.recv()).await {
Ok(Some(failure)) => failure,
Ok(None) => panic!("failures channel closed before a failure for id {id} arrived"),
Err(_) => panic!("timed out waiting for a liveness failure for id {id}"),
}
}
async fn assert_no_failure_within(rx: &mut mpsc::Receiver<LivenessFailure>, window: Duration) {
match tokio::time::timeout(window, rx.recv()).await {
Err(_) => {} Ok(Some(failure)) => panic!("unexpected liveness failure observed: {failure:?}"),
Ok(None) => panic!("failures channel disconnected while checking for no failure"),
}
}
#[test]
fn prober_is_dyn_compatible() {
let _: &dyn Prober = &ScriptedProber::new(vec![]);
}
#[tokio::test(start_paused = true)]
async fn three_consecutive_failures_report_at_exactly_three_intervals() {
let config = config_with_threshold(3);
let interval = config.interval.as_duration();
let timeout = config.timeout.as_duration();
let prober = Arc::new(ScriptedProber::new(vec![
Err(ProbeFailure::Timeout),
Err(ProbeFailure::Timeout),
Err(ProbeFailure::Timeout),
]));
let (tx, mut rx) = mpsc::channel(8);
let start = tokio::time::Instant::now();
let _handle =
spawn_and_settle(1, 100, config, Arc::clone(&prober) as Arc<dyn Prober>, tx).await;
let failure = expect_failure(&mut rx, 1).await;
assert_eq!(failure, LivenessFailure { id: 1, pid: 100 });
assert_eq!(
tokio::time::Instant::now() - start,
interval * 3,
"liveness failure should be reported at exactly the third probe, not before or after"
);
assert_eq!(
prober.last_timeout(),
timeout,
"the loop must pass config.timeout, not some other duration, to Prober::probe"
);
}
#[tokio::test(start_paused = true)]
async fn counter_re_accumulates_after_a_reset_and_trips_on_the_sixth_probe() {
let config = config_with_threshold(3);
let interval = config.interval.as_duration();
let prober = Arc::new(ScriptedProber::new(vec![
Err(ProbeFailure::Timeout),
Err(ProbeFailure::Timeout),
Ok(()),
Err(ProbeFailure::Timeout),
Err(ProbeFailure::Timeout),
Err(ProbeFailure::Timeout),
]));
let (tx, mut rx) = mpsc::channel(8);
let start = tokio::time::Instant::now();
let _handle =
spawn_and_settle(3, 300, config, Arc::clone(&prober) as Arc<dyn Prober>, tx).await;
let failure = expect_failure(&mut rx, 3).await;
assert_eq!(failure, LivenessFailure { id: 3, pid: 300 });
assert_eq!(
tokio::time::Instant::now() - start,
interval * 6,
"the sixth probe (three failures after the reset) should be the one that trips"
);
}
#[tokio::test(start_paused = true)]
async fn threshold_of_one_reports_after_a_single_failure() {
let config = config_with_threshold(1);
let interval = config.interval.as_duration();
let prober = Arc::new(ScriptedProber::new(vec![Err(ProbeFailure::Timeout)]));
let (tx, mut rx) = mpsc::channel(8);
let start = tokio::time::Instant::now();
let _handle =
spawn_and_settle(4, 400, config, Arc::clone(&prober) as Arc<dyn Prober>, tx).await;
let failure = expect_failure(&mut rx, 4).await;
assert_eq!(failure, LivenessFailure { id: 4, pid: 400 });
assert_eq!(
tokio::time::Instant::now() - start,
interval,
"threshold of one must report after exactly one probe, not two"
);
}
#[tokio::test(start_paused = true)]
async fn no_further_probing_after_a_failure_is_reported() {
let config = config_with_threshold(1);
let interval = config.interval.as_duration();
let prober = Arc::new(ScriptedProber::new(vec![Err(ProbeFailure::Timeout)]));
let (tx, mut rx) = mpsc::channel(8);
let _handle = spawn_and_settle(
5,
500,
config,
Arc::clone(&prober) as Arc<dyn Prober>,
tx.clone(),
)
.await;
expect_failure(&mut rx, 5).await;
let calls_after_report = prober.calls();
assert_no_failure_within(&mut rx, intervals(interval, 3)).await;
assert_eq!(
prober.calls(),
calls_after_report,
"prober must not be called again after the loop has reported and ended"
);
}
#[tokio::test(start_paused = true)]
async fn a_slow_probe_still_paces_by_interval_after_completion() {
let config = config_with_threshold(3); let interval = config.interval.as_duration();
let prober = Arc::new(ScriptedProber::new(vec![Ok(())]).with_delay(interval * 2));
let (tx, mut rx) = mpsc::channel(8);
let _handle =
spawn_and_settle(6, 600, config, Arc::clone(&prober) as Arc<dyn Prober>, tx).await;
assert_no_failure_within(&mut rx, intervals(interval, 12)).await;
assert_eq!(prober.calls(), 4);
}
#[tokio::test(start_paused = true)]
async fn aborting_the_handle_stops_probing() {
let config = config_with_threshold(1000); let interval = config.interval.as_duration();
let prober = Arc::new(ScriptedProber::new(vec![Err(ProbeFailure::Timeout)]));
let (tx, _rx) = mpsc::channel(8);
let handle =
spawn_and_settle(7, 700, config, Arc::clone(&prober) as Arc<dyn Prober>, tx).await;
tokio::time::sleep(intervals(interval, 2)).await;
let calls_before_abort = prober.calls();
assert!(
calls_before_abort > 0,
"expected at least one probe before abort"
);
handle.abort();
tokio::time::sleep(intervals(interval, 3)).await;
assert_eq!(
prober.calls(),
calls_before_abort,
"prober must not be called again once the handle has been aborted"
);
}
#[tokio::test(start_paused = true)]
async fn a_timeout_longer_than_the_interval_is_honoured_and_paces_the_next_probe() {
let config = ProbeConfig {
interval: shep_core::values::UpDuration::from_millis(2_000),
timeout: shep_core::values::UpDuration::from_millis(5_000),
..config_with_threshold(3)
};
let interval = config.interval.as_duration();
let timeout = config.timeout.as_duration();
let prober =
Arc::new(ScriptedProber::new(vec![Err(ProbeFailure::Timeout)]).with_delay(timeout));
let (tx, mut rx) = mpsc::channel(8);
let start = tokio::time::Instant::now();
let _handle =
spawn_and_settle(9, 900, config, Arc::clone(&prober) as Arc<dyn Prober>, tx).await;
let failure = expect_failure(&mut rx, 9).await;
assert_eq!(failure, LivenessFailure { id: 9, pid: 900 });
assert_eq!(
tokio::time::Instant::now() - start,
(interval + timeout) * 3,
"three probes that each sleep `interval` and then run for `timeout` must report \
at 3 x (interval + timeout)"
);
assert_eq!(
prober.last_timeout(),
timeout,
"the loop must pass config.timeout unclamped, even where it exceeds interval"
);
assert_eq!(prober.calls(), 3);
}
#[tokio::test(start_paused = true)]
async fn a_zero_interval_is_clamped_instead_of_hot_spinning() {
let config = ProbeConfig {
interval: shep_core::values::UpDuration::from_millis(0),
..config_with_threshold(1000) };
let prober = Arc::new(ScriptedProber::new(vec![Ok(())]));
let (tx, _rx) = mpsc::channel(8);
let _handle =
spawn_and_settle(8, 800, config, Arc::clone(&prober) as Arc<dyn Prober>, tx).await;
tokio::time::sleep(intervals(MIN_PROBE_INTERVAL, 5)).await;
assert_eq!(
prober.calls(),
5,
"interval must be clamped to MIN_PROBE_INTERVAL, not trusted as-is"
);
}
}