use core::time::Duration;
use std::sync::Arc;
use tokio::sync::oneshot;
use shep_core::config::{AppConfig, ProbeConfig, ProbeTarget, ProbeTargetError};
use super::Prober;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ReadinessSource {
Channel,
Probe(ProbeConfig, ProbeTarget),
Heuristic,
}
impl ReadinessSource {
pub fn of(config: &AppConfig) -> Result<Self, ProbeTargetError> {
if config.wait_ready {
return Ok(Self::Channel);
}
match &config.readiness_probe {
Some(probe) => Ok(Self::Probe(probe.clone(), ProbeTarget::parse(probe)?)),
None => Ok(Self::Heuristic),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Readiness {
Ready,
TimedOut,
}
pub async fn await_ready(
source: &ReadinessSource,
deadline: Duration,
channel: oneshot::Receiver<()>,
prober: Arc<dyn Prober>,
) -> Readiness {
match source {
ReadinessSource::Channel => {
let wait_for_signal = async {
if channel.await.is_err() {
core::future::pending::<()>().await;
}
};
match tokio::time::timeout(deadline, wait_for_signal).await {
Ok(()) => Readiness::Ready,
Err(_elapsed) => Readiness::TimedOut,
}
}
ReadinessSource::Probe(config, target) => {
let interval = config.interval.as_duration();
let timeout = config.timeout.as_duration();
let poll = async {
loop {
if prober.probe(target, timeout).await.is_ok() {
return;
}
tokio::time::sleep(interval).await;
}
};
match tokio::time::timeout(deadline, poll).await {
Ok(()) => Readiness::Ready,
Err(_elapsed) => Readiness::TimedOut,
}
}
ReadinessSource::Heuristic => {
tokio::time::sleep(deadline).await;
Readiness::Ready
}
}
}
#[cfg(test)]
mod tests {
use core::time::Duration;
use std::sync::Arc;
use tokio::sync::oneshot;
use super::*;
use crate::probes::ProbeFailure;
use crate::testing::{ScriptedProber, probe_config};
use shep_core::config::ProbeKind;
use shep_core::values::UpDuration;
fn unused_prober() -> Arc<dyn Prober> {
Arc::new(ScriptedProber::new(vec![]))
}
fn tcp_source(interval_ms: u64) -> ReadinessSource {
let config = ProbeConfig {
interval: UpDuration::from_millis(interval_ms),
..probe_config(ProbeKind::Tcp, "localhost:5432")
};
let target = ProbeTarget::parse(&config).expect("a valid tcp target parses");
ReadinessSource::Probe(config, target)
}
#[tokio::test(start_paused = true)]
async fn channel_ready_before_the_deadline_resolves_at_signal_time() {
let (tx, rx) = oneshot::channel();
let start = tokio::time::Instant::now();
tokio::spawn(async move {
tokio::time::sleep(Duration::from_millis(500)).await;
let _ = tx.send(());
});
let readiness = await_ready(
&ReadinessSource::Channel,
Duration::from_secs(3),
rx,
unused_prober(),
)
.await;
assert_eq!(readiness, Readiness::Ready);
assert_eq!(
tokio::time::Instant::now() - start,
Duration::from_millis(500)
);
}
#[tokio::test(start_paused = true)]
async fn channel_with_no_signal_times_out_at_exactly_the_deadline() {
let (_tx, rx) = oneshot::channel(); let start = tokio::time::Instant::now();
let deadline = Duration::from_secs(3);
let readiness = await_ready(&ReadinessSource::Channel, deadline, rx, unused_prober()).await;
assert_eq!(readiness, Readiness::TimedOut);
assert_eq!(tokio::time::Instant::now() - start, deadline);
}
#[tokio::test(start_paused = true)]
async fn channel_dropped_without_signalling_times_out_at_the_deadline_not_immediately() {
let (tx, rx) = oneshot::channel();
drop(tx);
let start = tokio::time::Instant::now();
let deadline = Duration::from_secs(3);
let readiness = await_ready(&ReadinessSource::Channel, deadline, rx, unused_prober()).await;
assert_eq!(readiness, Readiness::TimedOut);
assert_eq!(tokio::time::Instant::now() - start, deadline);
}
#[tokio::test(start_paused = true)]
async fn probe_becomes_ready_two_failures_then_a_pass_at_two_intervals() {
let source = tcp_source(1_000);
let (_tx, rx) = oneshot::channel();
let prober = Arc::new(ScriptedProber::new(vec![
Err(ProbeFailure::Timeout),
Err(ProbeFailure::Timeout),
Ok(()),
]));
let start = tokio::time::Instant::now();
let readiness = await_ready(&source, Duration::from_secs(10), rx, prober).await;
assert_eq!(readiness, Readiness::Ready);
assert_eq!(tokio::time::Instant::now() - start, Duration::from_secs(2));
}
#[tokio::test(start_paused = true)]
async fn probe_ready_immediately_when_the_first_probe_passes() {
let source = tcp_source(1_000);
let (_tx, rx) = oneshot::channel();
let prober = Arc::new(ScriptedProber::new(vec![Ok(())]));
let start = tokio::time::Instant::now();
let readiness = await_ready(&source, Duration::from_secs(10), rx, prober).await;
assert_eq!(readiness, Readiness::Ready);
assert_eq!(
tokio::time::Instant::now(),
start,
"the first probe must run at t=0, not after one interval"
);
}
#[tokio::test(start_paused = true)]
async fn probe_that_always_fails_times_out_at_the_deadline() {
let source = tcp_source(1_000);
let (_tx, rx) = oneshot::channel();
let prober = Arc::new(ScriptedProber::new(vec![Err(ProbeFailure::Timeout)]));
let start = tokio::time::Instant::now();
let deadline = Duration::from_secs(3);
let readiness = await_ready(&source, deadline, rx, prober).await;
assert_eq!(readiness, Readiness::TimedOut);
assert_eq!(tokio::time::Instant::now() - start, deadline);
}
#[tokio::test(start_paused = true)]
async fn heuristic_becomes_ready_when_its_deadline_elapses() {
let (_tx, rx) = oneshot::channel();
let start = tokio::time::Instant::now();
let deadline = Duration::from_secs(3);
let readiness =
await_ready(&ReadinessSource::Heuristic, deadline, rx, unused_prober()).await;
assert_eq!(readiness, Readiness::Ready);
assert_eq!(tokio::time::Instant::now() - start, deadline);
}
#[test]
fn wait_ready_wins_over_a_configured_probe() {
let mut app = AppConfig::minimal("web", "./srv");
app.wait_ready = true;
app.readiness_probe = Some(probe_config(ProbeKind::Tcp, "localhost:5432"));
assert_eq!(ReadinessSource::of(&app), Ok(ReadinessSource::Channel));
}
#[test]
fn a_configured_probe_is_used_when_wait_ready_is_unset() {
let mut app = AppConfig::minimal("web", "./srv");
app.readiness_probe = Some(probe_config(ProbeKind::Tcp, "localhost:5432"));
let target = ProbeTarget::parse(app.readiness_probe.as_ref().unwrap()).unwrap();
assert_eq!(
ReadinessSource::of(&app),
Ok(ReadinessSource::Probe(
app.readiness_probe.clone().unwrap(),
target
))
);
}
#[test]
fn neither_configured_is_heuristic() {
let app = AppConfig::minimal("web", "./srv");
assert_eq!(ReadinessSource::of(&app), Ok(ReadinessSource::Heuristic));
}
#[test]
fn a_malformed_probe_target_is_rejected() {
let mut app = AppConfig::minimal("web", "./srv");
let bad_probe = probe_config(ProbeKind::Http, "not-a-url");
app.readiness_probe = Some(bad_probe.clone());
let expected = ProbeTarget::parse(&bad_probe).unwrap_err();
assert_eq!(ReadinessSource::of(&app), Err(expected));
}
}