use std::collections::{BTreeMap, BTreeSet};
use std::sync::{Arc, Mutex, PoisonError};
use shep_core::selector::ProcessSelector;
use crate::supervisor::SupervisorHandle;
#[non_exhaustive]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Refusal {
Restart,
Stale,
AlreadyStale,
}
#[derive(Debug, Clone, Default)]
pub struct DogRefusals {
seen: Arc<Mutex<Links>>,
}
#[derive(Debug, Default)]
struct Links {
refusals: BTreeMap<String, u32>,
handshook: BTreeSet<String>,
}
impl DogRefusals {
#[must_use]
pub fn new() -> Self {
Self::default()
}
pub fn refused(&self, name: &str) -> Refusal {
let mut seen = self.lock();
seen.handshook.remove(name);
let count = seen.refusals.entry(name.to_string()).or_insert(0);
*count = count.saturating_add(1);
match *count {
1 => Refusal::Restart,
2 => Refusal::Stale,
_ => Refusal::AlreadyStale,
}
}
pub fn handshook(&self, name: &str) -> bool {
let mut seen = self.lock();
seen.refusals.remove(name);
seen.handshook.insert(name.to_string())
}
#[must_use]
pub fn has_handshook(&self, name: &str) -> bool {
self.lock().handshook.contains(name)
}
#[must_use]
pub fn restarting(&self) -> Vec<String> {
self.lock()
.refusals
.iter()
.filter(|(_, count)| **count == 1)
.map(|(name, _)| name.clone())
.collect()
}
#[must_use]
pub fn stale(&self) -> Vec<String> {
self.lock()
.refusals
.iter()
.filter(|(_, count)| **count >= 2)
.map(|(name, _)| name.clone())
.collect()
}
fn lock(&self) -> std::sync::MutexGuard<'_, Links> {
self.seen.lock().unwrap_or_else(PoisonError::into_inner)
}
}
pub fn record_refused_dog(
name: &str,
client_version: &str,
refusals: &DogRefusals,
supervisor: &SupervisorHandle,
) -> Refusal {
let verdict = refusals.refused(name);
match verdict {
Refusal::Restart => {
tracing::warn!(
dog = %name,
dog_version = %client_version,
"refused a dog on protocol skew; restarting it once from the binary on disk"
);
let supervisor = supervisor.clone();
let name = name.to_string();
tokio::spawn(async move { restart_refused_dog(&supervisor, &name).await });
}
Refusal::Stale => tracing::error!(
dog = %name,
dog_version = %client_version,
"refused a dog on protocol skew again after restarting it: the binary on disk speaks the same protocol the running one did, so this dog is stale and will not be restarted again. Rebuild or reinstall it against this shep"
),
Refusal::AlreadyStale => tracing::debug!(
dog = %name,
dog_version = %client_version,
"refused a dog already reported stale"
),
}
verdict
}
pub(super) async fn restart_refused_dog(supervisor: &SupervisorHandle, name: &str) {
match supervisor
.restart_automatic(ProcessSelector::Name(name.to_string()))
.await
{
Ok(_) => tracing::info!(dog = %name, "restarted a refused dog from the binary on disk"),
Err(err) => tracing::warn!(dog = %name, %err, "a refused dog could not be restarted"),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_refused_dog_earns_one_restart_and_is_then_stale_forever() {
let refusals = DogRefusals::new();
assert!(refusals.stale().is_empty());
assert_eq!(refusals.refused("metrics"), Refusal::Restart);
assert!(
refusals.stale().is_empty(),
"one refusal is a dog to restart, not a dog to give up on"
);
assert_eq!(refusals.refused("metrics"), Refusal::Stale);
assert_eq!(refusals.stale(), vec!["metrics".to_string()]);
for _ in 0..5 {
assert_eq!(refusals.refused("metrics"), Refusal::AlreadyStale);
}
assert_eq!(refusals.stale(), vec!["metrics".to_string()]);
}
#[test]
fn a_dog_that_gets_in_is_owed_a_fresh_restart_if_it_is_ever_refused_again() {
let refusals = DogRefusals::new();
assert_eq!(refusals.refused("metrics"), Refusal::Restart);
refusals.handshook("metrics");
assert!(refusals.stale().is_empty());
assert_eq!(
refusals.refused("metrics"),
Refusal::Restart,
"the restart that fixed it must not be charged against the next episode"
);
}
#[test]
fn each_dog_carries_its_own_count() {
let refusals = DogRefusals::new();
assert_eq!(refusals.refused("bark"), Refusal::Restart);
assert_eq!(refusals.refused("bark"), Refusal::Stale);
assert_eq!(
refusals.refused("metrics"),
Refusal::Restart,
"bark's two refusals are bark's"
);
refusals.handshook("metrics");
assert_eq!(
refusals.stale(),
vec!["bark".to_string()],
"one dog getting in says nothing about another"
);
}
#[test]
fn a_dog_mid_restart_is_neither_stale_nor_settled() {
let refusals = DogRefusals::new();
assert!(refusals.restarting().is_empty());
refusals.refused("metrics");
assert_eq!(refusals.restarting(), vec!["metrics".to_string()]);
assert!(refusals.stale().is_empty());
refusals.refused("metrics");
assert!(
refusals.restarting().is_empty(),
"a dog that has been given up on is settled, not still being restarted"
);
assert_eq!(refusals.stale(), vec!["metrics".to_string()]);
}
#[test]
fn only_an_accepted_handshake_says_a_dog_has_answered() {
let refusals = DogRefusals::new();
assert!(
!refusals.has_handshook("metrics"),
"a dog nobody has heard from has not answered"
);
refusals.handshook("metrics");
assert!(refusals.has_handshook("metrics"));
assert!(!refusals.has_handshook("bark"), "one dog answers for one");
refusals.refused("metrics");
assert!(
!refusals.has_handshook("metrics"),
"the handshake that earned the mark is the one that just died"
);
}
}