use core::time::Duration;
use std::collections::BTreeMap;
use tokio::time::Instant;
use shep_core::protocol::ProcessInfo;
use shep_core::status::ProcStatus;
use crate::bus::Bus;
use crate::supervisor::SupervisorHandle;
use super::PeerContacts;
use super::narrate::narrate;
use super::refusals::{DogRefusals, Refusal, restart_refused_dog};
use super::verdict::{Silence, first_rung_evidence, stale_verdict};
pub const DOG_SILENCE_BUDGET: Duration = Duration::from_secs(5);
const DOG_SILENCE_POLL: Duration = Duration::from_secs(1);
pub(crate) fn silent_dogs(infos: &[ProcessInfo], refusals: &DogRefusals) -> Vec<String> {
let stale = refusals.stale();
let mut names: Vec<String> = infos
.iter()
.filter(|info| {
info.dog.is_some()
&& matches!(info.status, ProcStatus::Starting | ProcStatus::Online)
&& !refusals.has_handshook(&info.name)
&& !stale.contains(&info.name)
})
.map(|info| info.name.clone())
.collect();
names.sort();
names.dedup();
names
}
#[derive(Debug, Default)]
pub(crate) struct SilentDogs {
first_seen: BTreeMap<String, Instant>,
}
impl SilentDogs {
fn due(&mut self, silent: &[String], now: Instant) -> Vec<String> {
self.first_seen.retain(|name, _| silent.contains(name));
let mut due = Vec::new();
for name in silent {
let since = self.first_seen.entry(name.clone()).or_insert(now);
if now.saturating_duration_since(*since) >= DOG_SILENCE_BUDGET {
*since = now;
due.push(name.clone());
}
}
due
}
}
pub(crate) async fn check_silent_dogs(
supervisor: &SupervisorHandle,
refusals: &DogRefusals,
contacts: &PeerContacts,
events: &Bus,
seen: &mut SilentDogs,
now: Instant,
) -> Vec<(String, Refusal)> {
if contacts.is_warming() {
return Vec::new();
}
let Ok(infos) = supervisor.list_checked().await else {
return Vec::new();
};
let silent = silent_dogs(&infos, refusals);
let mut acted = Vec::new();
for name in seen.due(&silent, now) {
let info = infos.iter().find(|info| info.name == name);
let evidence = Silence::of(info.and_then(|info| info.pid), contacts);
let verdict = record_silent_dog(&name, info, evidence, refusals, events, supervisor).await;
acted.push((name, verdict));
}
acted
}
async fn record_silent_dog(
name: &str,
info: Option<&ProcessInfo>,
evidence: Silence,
refusals: &DogRefusals,
events: &Bus,
supervisor: &SupervisorHandle,
) -> Refusal {
let verdict = refusals.refused(name);
match verdict {
Refusal::Restart => {
let seen = first_rung_evidence(evidence);
tracing::warn!(
dog = %name,
silent_for_secs = DOG_SILENCE_BUDGET.as_secs(),
evidence = %seen,
"a dog has been running without ever answering this shepherd; restarting it once from the binary on disk"
);
if let Some(info) = info {
narrate(
events,
info,
&format!(
"this dog has been running for {}s without ever answering this shepherd: {seen}. Restarting it once from the binary on disk",
DOG_SILENCE_BUDGET.as_secs()
),
)
.await;
}
restart_refused_dog(supervisor, name).await;
}
Refusal::Stale => {
let verdict = stale_verdict(name, evidence);
tracing::error!(dog = %name, "{verdict}");
if let Some(info) = info {
narrate(events, info, &verdict).await;
}
}
Refusal::AlreadyStale => tracing::debug!(
dog = %name,
"a silent dog that was already reported stale"
),
}
verdict
}
pub fn spawn_silent_dog_watch(
supervisor: SupervisorHandle,
refusals: DogRefusals,
contacts: PeerContacts,
events: Bus,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
let mut ticks = tokio::time::interval(DOG_SILENCE_POLL);
ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
let mut seen = SilentDogs::default();
loop {
let now = ticks.tick().await;
check_silent_dogs(&supervisor, &refusals, &contacts, &events, &mut seen, now).await;
}
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::fake::ProcScript;
use super::super::contacts::PEER_CONTACT_WARMUP;
use super::super::test_support::start_test_dog;
const SETTLE_STEP: Duration = Duration::from_millis(250);
const LADDER_BUDGET: Duration = Duration::from_secs(45);
async fn settle_until(
what: &str,
within: Duration,
mut settled: impl FnMut() -> bool,
) -> Duration {
let began = Instant::now();
tokio::time::timeout(within, async {
while !settled() {
tokio::time::sleep(SETTLE_STEP).await;
}
})
.await
.unwrap_or_else(|_| panic!("{what} did not happen within {within:?} of virtual time"));
began.elapsed()
}
#[tokio::test(start_paused = true)]
async fn a_dog_that_never_answers_is_restarted_once_and_then_marked_stale() {
let h = crate::testing::harness(vec![
ProcScript::never_exits(),
ProcScript::never_exits(),
ProcScript::never_exits(),
ProcScript::never_exits(),
]);
start_test_dog(&h.ctx, "metrics").await;
tokio::time::advance(PEER_CONTACT_WARMUP * 2).await;
let refusals = &h.ctx.dog_refusals;
let contacts = &h.ctx.peer_contacts;
let events = &h.ctx.events;
let mut seen = SilentDogs::default();
let t0 = Instant::now();
assert!(
check_silent_dogs(&h.ctx.supervisor, refusals, contacts, events, &mut seen, t0)
.await
.is_empty(),
"a dog seen quiet for the first time has not yet been quiet for any length of time"
);
assert_eq!(
check_silent_dogs(
&h.ctx.supervisor,
refusals,
contacts,
events,
&mut seen,
t0 + DOG_SILENCE_BUDGET
)
.await,
vec![("metrics".to_string(), Refusal::Restart)],
"a whole budget of silence buys the one restart from disk"
);
assert_eq!(refusals.restarting(), vec!["metrics".to_string()]);
assert!(
refusals.stale().is_empty(),
"one silence is a dog to restart, not a dog to give up on"
);
assert_eq!(
check_silent_dogs(
&h.ctx.supervisor,
refusals,
contacts,
events,
&mut seen,
t0 + 2 * DOG_SILENCE_BUDGET
)
.await,
vec![("metrics".to_string(), Refusal::Stale)],
"the restart ran and the dog still has not spoken, so the ladder ends here"
);
assert_eq!(refusals.stale(), vec!["metrics".to_string()]);
assert!(
check_silent_dogs(
&h.ctx.supervisor,
refusals,
contacts,
events,
&mut seen,
t0 + 3 * DOG_SILENCE_BUDGET
)
.await
.is_empty(),
"a dog already given up on is not laddered again, however long it stays quiet"
);
}
#[tokio::test(start_paused = true)]
async fn a_dog_that_answers_inside_the_budget_is_never_touched() {
let h = crate::testing::harness(vec![ProcScript::never_exits()]);
start_test_dog(&h.ctx, "metrics").await;
let refusals = &h.ctx.dog_refusals;
let contacts = &h.ctx.peer_contacts;
let events = &h.ctx.events;
refusals.handshook("metrics");
let mut seen = SilentDogs::default();
let t0 = Instant::now();
for elapsed in [0, 1, 2, 10] {
assert!(
check_silent_dogs(
&h.ctx.supervisor,
refusals,
contacts,
events,
&mut seen,
t0 + elapsed * DOG_SILENCE_BUDGET
)
.await
.is_empty(),
"a dog this shepherd has heard from is not silent at any point on the clock"
);
}
assert!(refusals.restarting().is_empty());
assert!(refusals.stale().is_empty());
}
#[tokio::test(start_paused = true)]
async fn a_dog_already_stale_is_not_laddered_again() {
let h = crate::testing::harness(vec![ProcScript::never_exits()]);
start_test_dog(&h.ctx, "metrics").await;
let refusals = &h.ctx.dog_refusals;
let contacts = &h.ctx.peer_contacts;
let events = &h.ctx.events;
refusals.refused("metrics");
refusals.refused("metrics");
assert_eq!(refusals.stale(), vec!["metrics".to_string()]);
let mut seen = SilentDogs::default();
let t0 = Instant::now();
for elapsed in [0, 1, 2, 5] {
assert!(
check_silent_dogs(
&h.ctx.supervisor,
refusals,
contacts,
events,
&mut seen,
t0 + elapsed * DOG_SILENCE_BUDGET
)
.await
.is_empty(),
"the ladder ends at stale; there is no rung after it to reach"
);
}
}
#[tokio::test(start_paused = true)]
async fn asking_repeatedly_does_not_advance_the_ladder() {
let h = crate::testing::harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
start_test_dog(&h.ctx, "metrics").await;
tokio::time::advance(PEER_CONTACT_WARMUP * 2).await;
let refusals = &h.ctx.dog_refusals;
let contacts = &h.ctx.peer_contacts;
let events = &h.ctx.events;
let mut seen = SilentDogs::default();
let t0 = Instant::now();
for look in 0..20 {
assert!(
check_silent_dogs(
&h.ctx.supervisor,
refusals,
contacts,
events,
&mut seen,
t0 + (DOG_SILENCE_BUDGET / 20) * look
)
.await
.is_empty(),
"look {look} fell inside the budget and must not have moved the dog along"
);
}
assert!(refusals.restarting().is_empty());
assert_eq!(
check_silent_dogs(
&h.ctx.supervisor,
refusals,
contacts,
events,
&mut seen,
t0 + DOG_SILENCE_BUDGET
)
.await,
vec![("metrics".to_string(), Refusal::Restart)],
"the clock is what moves the dog along, and it has now moved"
);
}
#[tokio::test(start_paused = true)]
async fn a_dog_that_never_calls_still_earns_its_rebuild_after_the_warm_up() {
let map_started_no_earlier_than = Instant::now();
let h = crate::testing::harness(vec![
ProcScript::never_exits(),
ProcScript::never_exits(),
ProcScript::never_exits(),
]);
start_test_dog(&h.ctx, "metrics").await;
let refusals = h.ctx.dog_refusals.clone();
let contacts = h.ctx.peer_contacts.clone();
assert!(contacts.is_warming(), "a fresh map starts cold");
let watch = spawn_silent_dog_watch(
h.ctx.supervisor.clone(),
refusals.clone(),
contacts.clone(),
h.ctx.events.clone(),
);
let restart_rung = settle_until("the silent dog's restart rung", LADDER_BUDGET, || {
!refusals.restarting().is_empty()
})
.await;
assert_eq!(
refusals.restarting(),
vec!["metrics".to_string()],
"the dog nothing ever connected from is the one that earns the rung"
);
let first_rung_at = map_started_no_earlier_than.elapsed();
assert!(
first_rung_at >= PEER_CONTACT_WARMUP + DOG_SILENCE_BUDGET,
"a cold map must judge nothing: the first rung landed {first_rung_at:?} in, \
which is inside the {PEER_CONTACT_WARMUP:?} warm-up plus one \
{DOG_SILENCE_BUDGET:?} budget of silence it has to wait out"
);
assert!(
restart_rung >= DOG_SILENCE_BUDGET,
"no rung can be earned in less than a whole budget of silence: {restart_rung:?}"
);
settle_until("the silent dog's stale rung", LADDER_BUDGET, || {
refusals.stale().contains(&"metrics".to_string())
})
.await;
let info = h
.ctx
.supervisor
.list()
.await
.into_iter()
.find(|info| info.name == "metrics")
.expect("the dog fixture is listed");
let verdict = stale_verdict("metrics", Silence::of(info.pid, &contacts));
assert!(
verdict.contains("cannot reach this shep"),
"the earned rebuild advice must survive the warm-up: {verdict}"
);
watch.abort();
}
#[tokio::test(start_paused = true)]
async fn the_watcher_restarts_a_silent_dog_after_one_budget_of_paused_time() {
let h = crate::testing::harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
start_test_dog(&h.ctx, "metrics").await;
tokio::time::advance(PEER_CONTACT_WARMUP * 2).await;
let refusals = h.ctx.dog_refusals.clone();
let watch = spawn_silent_dog_watch(
h.ctx.supervisor.clone(),
refusals.clone(),
h.ctx.peer_contacts.clone(),
h.ctx.events.clone(),
);
let waited = settle_until("the silent dog's restart", LADDER_BUDGET, || {
!refusals.restarting().is_empty()
})
.await;
assert_eq!(
refusals.restarting(),
vec!["metrics".to_string()],
"one budget of silence, driven through the watcher's own tick, must earn exactly one restart"
);
assert!(
waited >= DOG_SILENCE_BUDGET,
"a restart is earned by a whole budget of silence, not by less: {waited:?}"
);
assert!(
refusals.stale().is_empty(),
"one silence is a dog to restart, not a dog to give up on"
);
watch.abort();
}
}