use core::fmt;
use core::time::Duration;
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex, PoisonError};
use shep_core::barks::{self, Bark};
use shep_core::config::{AppConfig, DaemonConfig, ResolvedApp, normalize};
use shep_core::paths::ShepPaths;
use shep_core::protocol::{BusEvent, DogSource, ProcessEventKind, ProcessInfo};
use shep_core::selector::ProcessSelector;
use shep_core::status::ProcStatus;
use tokio::sync::broadcast::{self, error::RecvError};
use tokio::time::Instant;
use crate::bus::SharedEvent;
use crate::supervisor::SupervisorHandle;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DogSpec {
pub name: String,
pub source: DogSource,
}
#[non_exhaustive]
#[derive(Debug)]
pub enum DogError {
NoBinary(std::io::Error),
UnsupportedSource(String),
Config(String),
Io(std::io::Error),
}
impl fmt::Display for DogError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::NoBinary(err) => write!(f, "this binary's own path is unresolvable: {err}"),
Self::UnsupportedSource(source) => {
write!(f, "no way to spawn a dog from source {source}")
}
Self::Config(msg) => write!(f, "dog configuration is unusable: {msg}"),
Self::Io(err) => write!(f, "dog configuration could not be read: {err}"),
}
}
}
impl core::error::Error for DogError {
fn source(&self) -> Option<&(dyn core::error::Error + 'static)> {
match self {
Self::NoBinary(err) | Self::Io(err) => Some(err),
Self::UnsupportedSource(_) | Self::Config(_) => None,
}
}
}
#[cfg(unix)]
fn builtin_program() -> Result<PathBuf, DogError> {
crate::handover::exec_target().map_err(DogError::NoBinary)
}
#[cfg(windows)]
fn builtin_program() -> Result<PathBuf, DogError> {
std::env::current_exe().map_err(DogError::NoBinary)
}
pub fn dog_app(spec: &DogSpec, paths: &ShepPaths) -> Result<ResolvedApp, DogError> {
let (script, args) = match &spec.source {
DogSource::BuiltIn => (
builtin_program()?.display().to_string(),
vec!["dog".to_string(), spec.name.clone()],
),
DogSource::Adopted { path } => (path.clone(), Vec::new()),
source => return Err(DogError::UnsupportedSource(format!("{source:?}"))),
};
let mut config = AppConfig::minimal(&spec.name, &script);
config.args = args;
config
.env
.insert("SHEP_HOME".to_string(), paths.home.display().to_string());
config
.env
.insert("SHEP_DOG_NAME".to_string(), spec.name.clone());
normalize(config).map_err(|err| DogError::Config(err.to_string()))
}
pub async fn spawn_enabled_dogs(
specs: &[DogSpec],
paths: &ShepPaths,
supervisor: &SupervisorHandle,
) {
for spec in specs {
let app = match dog_app(spec, paths) {
Ok(app) => app,
Err(err) => {
tracing::warn!(dog = %spec.name, %err, "a dog did not start");
continue;
}
};
match supervisor.start_dog(app, spec.source.clone()).await {
Ok(info) if info.dog.is_none() => tracing::warn!(
dog = %spec.name,
"a sheep is already registered under this name; the dog did not start"
),
Ok(_) => {}
Err(err) => tracing::warn!(dog = %spec.name, %err, "a dog did not start"),
}
}
}
pub fn dog_section(path: &Path, name: &str) -> Result<String, DogError> {
let source = match std::fs::read_to_string(path) {
Ok(source) => source,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(String::new()),
Err(err) => return Err(DogError::Io(err)),
};
let config = DaemonConfig::load(Some(&source), &|_| None)
.map_err(|err| DogError::Config(err.to_string()))?;
match config.dog.get(name) {
None => Ok(String::new()),
Some(table) => toml::to_string(table).map_err(|err| DogError::Config(err.to_string())),
}
}
#[non_exhaustive]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Refusal {
Restart,
Stale,
AlreadyStale,
}
#[derive(Debug, Clone, Default)]
pub struct DogRefusals {
seen: Arc<Mutex<Links>>,
}
#[derive(Debug, Default)]
struct Links {
refusals: BTreeMap<String, u32>,
handshook: BTreeSet<String>,
}
impl DogRefusals {
#[must_use]
pub fn new() -> Self {
Self::default()
}
pub fn refused(&self, name: &str) -> Refusal {
let mut seen = self.lock();
seen.handshook.remove(name);
let count = seen.refusals.entry(name.to_string()).or_insert(0);
*count = count.saturating_add(1);
match *count {
1 => Refusal::Restart,
2 => Refusal::Stale,
_ => Refusal::AlreadyStale,
}
}
pub fn handshook(&self, name: &str) {
let mut seen = self.lock();
seen.refusals.remove(name);
seen.handshook.insert(name.to_string());
}
#[must_use]
pub fn has_handshook(&self, name: &str) -> bool {
self.lock().handshook.contains(name)
}
#[must_use]
pub fn restarting(&self) -> Vec<String> {
self.lock()
.refusals
.iter()
.filter(|(_, count)| **count == 1)
.map(|(name, _)| name.clone())
.collect()
}
#[must_use]
pub fn stale(&self) -> Vec<String> {
self.lock()
.refusals
.iter()
.filter(|(_, count)| **count >= 2)
.map(|(name, _)| name.clone())
.collect()
}
fn lock(&self) -> std::sync::MutexGuard<'_, Links> {
self.seen.lock().unwrap_or_else(PoisonError::into_inner)
}
}
pub fn record_refused_dog(
name: &str,
client_version: &str,
refusals: &DogRefusals,
supervisor: &SupervisorHandle,
) -> Refusal {
let verdict = refusals.refused(name);
match verdict {
Refusal::Restart => {
tracing::warn!(
dog = %name,
dog_version = %client_version,
"refused a dog on protocol skew; restarting it once from the binary on disk"
);
let supervisor = supervisor.clone();
let name = name.to_string();
tokio::spawn(async move { restart_refused_dog(&supervisor, &name).await });
}
Refusal::Stale => tracing::error!(
dog = %name,
dog_version = %client_version,
"refused a dog on protocol skew again after restarting it: the binary on disk speaks the same protocol the running one did, so this dog is stale and will not be restarted again. Rebuild or reinstall it against this shep"
),
Refusal::AlreadyStale => tracing::debug!(
dog = %name,
dog_version = %client_version,
"refused a dog already reported stale"
),
}
verdict
}
async fn restart_refused_dog(supervisor: &SupervisorHandle, name: &str) {
match supervisor
.restart_automatic(ProcessSelector::Name(name.to_string()))
.await
{
Ok(_) => tracing::info!(dog = %name, "restarted a refused dog from the binary on disk"),
Err(err) => tracing::warn!(dog = %name, %err, "a refused dog could not be restarted"),
}
}
pub const DOG_SILENCE_BUDGET: Duration = Duration::from_secs(5);
const DOG_SILENCE_POLL: Duration = Duration::from_secs(1);
pub(crate) fn silent_dogs(infos: &[ProcessInfo], refusals: &DogRefusals) -> Vec<String> {
let stale = refusals.stale();
let mut names: Vec<String> = infos
.iter()
.filter(|info| {
info.dog.is_some()
&& matches!(info.status, ProcStatus::Starting | ProcStatus::Online)
&& !refusals.has_handshook(&info.name)
&& !stale.contains(&info.name)
})
.map(|info| info.name.clone())
.collect();
names.sort();
names.dedup();
names
}
#[derive(Debug, Default)]
struct SilentDogs {
first_seen: BTreeMap<String, Instant>,
}
impl SilentDogs {
fn due(&mut self, silent: &[String], now: Instant) -> Vec<String> {
self.first_seen.retain(|name, _| silent.contains(name));
let mut due = Vec::new();
for name in silent {
let since = self.first_seen.entry(name.clone()).or_insert(now);
if now.saturating_duration_since(*since) >= DOG_SILENCE_BUDGET {
*since = now;
due.push(name.clone());
}
}
due
}
}
async fn check_silent_dogs(
supervisor: &SupervisorHandle,
refusals: &DogRefusals,
seen: &mut SilentDogs,
now: Instant,
) -> Vec<(String, Refusal)> {
let Ok(infos) = supervisor.list_checked().await else {
return Vec::new();
};
let silent = silent_dogs(&infos, refusals);
let mut acted = Vec::new();
for name in seen.due(&silent, now) {
let verdict = record_silent_dog(&name, refusals, supervisor).await;
acted.push((name, verdict));
}
acted
}
async fn record_silent_dog(
name: &str,
refusals: &DogRefusals,
supervisor: &SupervisorHandle,
) -> Refusal {
let verdict = refusals.refused(name);
match verdict {
Refusal::Restart => {
tracing::warn!(
dog = %name,
silent_for_secs = DOG_SILENCE_BUDGET.as_secs(),
"a dog has been running without ever answering this shepherd; restarting it once from the binary on disk"
);
restart_refused_dog(supervisor, name).await;
}
Refusal::Stale => tracing::error!(
dog = %name,
"a dog restarted for never answering this shepherd has still not answered it: the binary on disk cannot talk to this shep either, so this dog is stale and will not be restarted again. Read its own log with `shep bleats`, then rebuild or reinstall it and restart it"
),
Refusal::AlreadyStale => tracing::debug!(
dog = %name,
"a silent dog that was already reported stale"
),
}
verdict
}
pub fn spawn_silent_dog_watch(
supervisor: SupervisorHandle,
refusals: DogRefusals,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
let mut ticks = tokio::time::interval(DOG_SILENCE_POLL);
ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
let mut seen = SilentDogs::default();
loop {
let now = ticks.tick().await;
check_silent_dogs(&supervisor, &refusals, &mut seen, now).await;
}
})
}
pub fn spawn_dog_watch(
mut events: broadcast::Receiver<SharedEvent>,
barks: PathBuf,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
loop {
match events.recv().await {
Ok(event) => {
if let BusEvent::Process {
event: ProcessEventKind::Errored,
info,
..
} = &*event
&& info.dog.is_some()
{
record_dog_errored(&barks, &info.name, info.restarts);
}
}
Err(RecvError::Lagged(count)) => {
tracing::warn!(
count,
"the shepherd's dog watch dropped bus events; a dog's exhausted restart budget may have gone unrecorded"
);
}
Err(RecvError::Closed) => break,
}
}
})
}
fn record_dog_errored(barks_path: &Path, name: &str, restarts: u32) {
let budget = AppConfig::default().max_restarts;
tracing::error!(dog = %name, restarts, budget, "a dog exhausted its restart budget");
let bark = Bark {
at_ms: crate::now_ms(),
rule: "daemon".to_string(),
subject: name.to_string(),
message: format!(
"dog {name} exhausted its restart budget: {restarts} restarts against a budget of {budget}"
),
sinks: Vec::new(),
};
if let Err(err) = barks::append(barks_path, &bark, barks::DEFAULT_MAX_BYTES) {
tracing::warn!(%err, dog = %name, "failed to record a dog's exhausted restart budget");
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::fake::ProcScript;
use crate::testing::test_paths;
use shep_core::protocol::ProcessInfo;
use shep_core::status::ProcStatus;
#[cfg(unix)]
#[test]
fn a_deleted_inode_answer_from_current_exe_never_becomes_a_dogs_script() {
let refusal = crate::handover::resolve_target(
[None, Some(PathBuf::from("/opt/shep/shep (deleted)"))],
None,
)
.unwrap_err();
let err = DogError::NoBinary(refusal);
assert_eq!(
err.to_string(),
"this binary's own path is unresolvable: no binary to exec: \
/opt/shep/shep (deleted) (names a deleted inode, not a file)"
);
}
#[test]
fn a_dogs_child_environment_carries_shep_home_and_its_name_and_no_configuration() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
std::fs::write(
&paths.daemon_config,
"[dog.bark]\nwebhook = \"https://example.invalid/hook\"\n",
)
.unwrap();
let spec = DogSpec {
name: "bark".to_string(),
source: DogSource::BuiltIn,
};
let app = dog_app(&spec, &paths).unwrap();
let assembled = crate::assemble::assemble(&app, 0, &paths, None);
assert_eq!(
assembled.env.get("SHEP_HOME"),
Some(&paths.home.display().to_string())
);
assert_eq!(
assembled.env.get("SHEP_DOG_NAME"),
Some(&"bark".to_string()),
"a dog is told the name its own section lives under"
);
assert!(
!assembled
.env
.values()
.any(|v| v.contains("example.invalid")),
"a dog's configuration never travels in its environment: {:?}",
assembled.env
);
}
#[test]
fn a_built_in_dog_runs_this_binary_and_an_adopted_one_runs_its_own() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let built_in = dog_app(
&DogSpec {
name: "metrics".to_string(),
source: DogSource::BuiltIn,
},
&paths,
)
.unwrap();
assert_eq!(
built_in.config().script,
std::env::current_exe().unwrap().display().to_string()
);
assert_eq!(built_in.config().args, vec!["dog", "metrics"]);
let adopted = dog_app(
&DogSpec {
name: "otel".to_string(),
source: DogSource::Adopted {
path: "/usr/local/bin/shep-otel".to_string(),
},
},
&paths,
)
.unwrap();
assert_eq!(adopted.config().script, "/usr/local/bin/shep-otel");
assert!(adopted.config().args.is_empty());
assert_eq!(
adopted.config().name,
"otel",
"the NAME is the config key, never the filename"
);
}
#[test]
fn an_adopted_dog_is_told_the_name_it_was_registered_under() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let adopted = dog_app(
&DogSpec {
name: "telemetry".to_string(),
source: DogSource::Adopted {
path: "/usr/local/bin/shep-otel".to_string(),
},
},
&paths,
)
.unwrap();
assert!(
adopted.config().args.is_empty(),
"the name arrives without shep inventing an argv for a foreign binary"
);
assert_eq!(
adopted.config().env.get("SHEP_DOG_NAME"),
Some(&"telemetry".to_string())
);
}
#[test]
fn a_dogs_section_comes_back_as_its_own_table_and_nothing_else() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("shep.toml");
std::fs::write(
&path,
"[daemon]\nlog_json = true\n\n[dog.bark]\ndebounce = \"30s\"\n\n[dog.metrics]\nport = 9615\n",
)
.unwrap();
let bark = dog_section(&path, "bark").unwrap();
assert!(bark.contains("debounce"));
assert!(
!bark.contains("9615"),
"one dog never sees another's config"
);
assert!(!bark.contains("log_json"), "nor the daemon's own");
let parsed: toml::Table = toml::from_str(&bark).unwrap();
assert_eq!(parsed["debounce"].as_str(), Some("30s"));
assert_eq!(dog_section(&path, "absent").unwrap(), "");
assert_eq!(
dog_section(&dir.path().join("gone.toml"), "bark").unwrap(),
""
);
}
fn process_event(name: &str, kind: ProcessEventKind, dog: Option<DogSource>) -> SharedEvent {
SharedEvent::new(BusEvent::Process {
event: kind,
info: ProcessInfo::builder(1, name, ProcStatus::Errored)
.restarts(16)
.dog(dog)
.build(),
manually: false,
at_ms: 1_700_000_000_000,
})
}
fn errored_event(name: &str, dog: Option<DogSource>) -> SharedEvent {
process_event(name, ProcessEventKind::Errored, dog)
}
async fn await_barks(path: &std::path::Path, n: usize) -> Vec<Bark> {
tokio::time::timeout(std::time::Duration::from_secs(2), async {
loop {
let found = barks::read(path).unwrap_or_default();
if found.len() >= n {
return found;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
})
.await
.expect("barks.jsonl never reached the expected record count")
}
#[tokio::test]
async fn the_shepherd_records_a_dog_that_gave_up_and_leaves_the_sheep_to_bark() {
let dir = tempfile::tempdir().unwrap();
let barks_path = dir.path().join("barks.jsonl");
let (events, rx) = crate::bus::test_bus(16);
let watch = spawn_dog_watch(rx, barks_path.clone());
events.send(errored_event("web", None)).unwrap();
events
.send(errored_event("bark", Some(DogSource::BuiltIn)))
.unwrap();
let recorded = await_barks(&barks_path, 1).await;
assert_eq!(recorded.len(), 1, "one record, and it is the dog's");
assert_eq!(recorded[0].subject, "bark");
assert_eq!(recorded[0].rule, "daemon");
assert!(
recorded[0].sinks.is_empty(),
"the shepherd has no sinks and says so by carrying none"
);
watch.abort();
}
#[tokio::test]
async fn a_dog_that_merely_exited_is_not_recorded() {
let dir = tempfile::tempdir().unwrap();
let barks_path = dir.path().join("barks.jsonl");
let (events, rx) = crate::bus::test_bus(16);
let watch = spawn_dog_watch(rx, barks_path.clone());
events
.send(process_event(
"bark",
ProcessEventKind::Exit,
Some(DogSource::BuiltIn),
))
.unwrap();
events
.send(errored_event("bark", Some(DogSource::BuiltIn)))
.unwrap();
let recorded = await_barks(&barks_path, 1).await;
assert_eq!(
recorded.len(),
1,
"the Exit left no record; only the Errored that followed it did"
);
watch.abort();
}
#[test]
fn a_refused_dog_earns_one_restart_and_is_then_stale_forever() {
let refusals = DogRefusals::new();
assert!(refusals.stale().is_empty());
assert_eq!(refusals.refused("metrics"), Refusal::Restart);
assert!(
refusals.stale().is_empty(),
"one refusal is a dog to restart, not a dog to give up on"
);
assert_eq!(refusals.refused("metrics"), Refusal::Stale);
assert_eq!(refusals.stale(), vec!["metrics".to_string()]);
for _ in 0..5 {
assert_eq!(refusals.refused("metrics"), Refusal::AlreadyStale);
}
assert_eq!(refusals.stale(), vec!["metrics".to_string()]);
}
#[test]
fn a_dog_that_gets_in_is_owed_a_fresh_restart_if_it_is_ever_refused_again() {
let refusals = DogRefusals::new();
assert_eq!(refusals.refused("metrics"), Refusal::Restart);
refusals.handshook("metrics");
assert!(refusals.stale().is_empty());
assert_eq!(
refusals.refused("metrics"),
Refusal::Restart,
"the restart that fixed it must not be charged against the next episode"
);
}
#[test]
fn each_dog_carries_its_own_count() {
let refusals = DogRefusals::new();
assert_eq!(refusals.refused("bark"), Refusal::Restart);
assert_eq!(refusals.refused("bark"), Refusal::Stale);
assert_eq!(
refusals.refused("metrics"),
Refusal::Restart,
"bark's two refusals are bark's"
);
refusals.handshook("metrics");
assert_eq!(
refusals.stale(),
vec!["bark".to_string()],
"one dog getting in says nothing about another"
);
}
#[test]
fn a_dog_mid_restart_is_neither_stale_nor_settled() {
let refusals = DogRefusals::new();
assert!(refusals.restarting().is_empty());
refusals.refused("metrics");
assert_eq!(refusals.restarting(), vec!["metrics".to_string()]);
assert!(refusals.stale().is_empty());
refusals.refused("metrics");
assert!(
refusals.restarting().is_empty(),
"a dog that has been given up on is settled, not still being restarted"
);
assert_eq!(refusals.stale(), vec!["metrics".to_string()]);
}
#[test]
fn only_an_accepted_handshake_says_a_dog_has_answered() {
let refusals = DogRefusals::new();
assert!(
!refusals.has_handshook("metrics"),
"a dog nobody has heard from has not answered"
);
refusals.handshook("metrics");
assert!(refusals.has_handshook("metrics"));
assert!(!refusals.has_handshook("bark"), "one dog answers for one");
refusals.refused("metrics");
assert!(
!refusals.has_handshook("metrics"),
"the handshake that earned the mark is the one that just died"
);
}
const DOG_FIXTURE_START_BUDGET: Duration = Duration::from_secs(10);
async fn start_test_dog(ctx: &crate::rpc::RpcContext, name: &str) {
let spec = DogSpec {
name: name.to_string(),
source: DogSource::BuiltIn,
};
let app = dog_app(&spec, &ctx.paths).expect("the dog fixture must assemble");
tokio::time::timeout(
DOG_FIXTURE_START_BUDGET,
ctx.supervisor.start_dog(app, DogSource::BuiltIn),
)
.await
.expect("the dog fixture must start inside its budget")
.expect("the dog fixture must start");
}
#[tokio::test(start_paused = true)]
async fn a_dog_that_never_answers_is_restarted_once_and_then_marked_stale() {
let h = crate::testing::harness(vec![
ProcScript::never_exits(),
ProcScript::never_exits(),
ProcScript::never_exits(),
ProcScript::never_exits(),
]);
start_test_dog(&h.ctx, "metrics").await;
let refusals = &h.ctx.dog_refusals;
let mut seen = SilentDogs::default();
let t0 = Instant::now();
assert!(
check_silent_dogs(&h.ctx.supervisor, refusals, &mut seen, t0)
.await
.is_empty(),
"a dog seen quiet for the first time has not yet been quiet for any length of time"
);
assert_eq!(
check_silent_dogs(
&h.ctx.supervisor,
refusals,
&mut seen,
t0 + DOG_SILENCE_BUDGET
)
.await,
vec![("metrics".to_string(), Refusal::Restart)],
"a whole budget of silence buys the one restart from disk"
);
assert_eq!(refusals.restarting(), vec!["metrics".to_string()]);
assert!(
refusals.stale().is_empty(),
"one silence is a dog to restart, not a dog to give up on"
);
assert_eq!(
check_silent_dogs(
&h.ctx.supervisor,
refusals,
&mut seen,
t0 + 2 * DOG_SILENCE_BUDGET
)
.await,
vec![("metrics".to_string(), Refusal::Stale)],
"the restart ran and the dog still has not spoken: the binary on disk cannot talk to this shep either"
);
assert_eq!(refusals.stale(), vec!["metrics".to_string()]);
assert!(
check_silent_dogs(
&h.ctx.supervisor,
refusals,
&mut seen,
t0 + 3 * DOG_SILENCE_BUDGET
)
.await
.is_empty(),
"a dog already given up on is not laddered again, however long it stays quiet"
);
}
#[tokio::test(start_paused = true)]
async fn a_dog_that_answers_inside_the_budget_is_never_touched() {
let h = crate::testing::harness(vec![ProcScript::never_exits()]);
start_test_dog(&h.ctx, "metrics").await;
let refusals = &h.ctx.dog_refusals;
refusals.handshook("metrics");
let mut seen = SilentDogs::default();
let t0 = Instant::now();
for elapsed in [0, 1, 2, 10] {
assert!(
check_silent_dogs(
&h.ctx.supervisor,
refusals,
&mut seen,
t0 + elapsed * DOG_SILENCE_BUDGET
)
.await
.is_empty(),
"a dog this shepherd has heard from is not silent at any point on the clock"
);
}
assert!(refusals.restarting().is_empty());
assert!(refusals.stale().is_empty());
}
#[tokio::test(start_paused = true)]
async fn a_dog_already_stale_is_not_laddered_again() {
let h = crate::testing::harness(vec![ProcScript::never_exits()]);
start_test_dog(&h.ctx, "metrics").await;
let refusals = &h.ctx.dog_refusals;
refusals.refused("metrics");
refusals.refused("metrics");
assert_eq!(refusals.stale(), vec!["metrics".to_string()]);
let mut seen = SilentDogs::default();
let t0 = Instant::now();
for elapsed in [0, 1, 2, 5] {
assert!(
check_silent_dogs(
&h.ctx.supervisor,
refusals,
&mut seen,
t0 + elapsed * DOG_SILENCE_BUDGET
)
.await
.is_empty(),
"the ladder ends at stale; there is no rung after it to reach"
);
}
}
#[tokio::test(start_paused = true)]
async fn asking_repeatedly_does_not_advance_the_ladder() {
let h = crate::testing::harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
start_test_dog(&h.ctx, "metrics").await;
let refusals = &h.ctx.dog_refusals;
let mut seen = SilentDogs::default();
let t0 = Instant::now();
for look in 0..20 {
assert!(
check_silent_dogs(
&h.ctx.supervisor,
refusals,
&mut seen,
t0 + (DOG_SILENCE_BUDGET / 20) * look
)
.await
.is_empty(),
"look {look} fell inside the budget and must not have moved the dog along"
);
}
assert!(refusals.restarting().is_empty());
assert_eq!(
check_silent_dogs(
&h.ctx.supervisor,
refusals,
&mut seen,
t0 + DOG_SILENCE_BUDGET
)
.await,
vec![("metrics".to_string(), Refusal::Restart)],
"the clock is what moves the dog along, and it has now moved"
);
}
#[tokio::test(start_paused = true)]
async fn the_watcher_restarts_a_silent_dog_after_one_budget_of_paused_time() {
let h = crate::testing::harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
start_test_dog(&h.ctx, "metrics").await;
let refusals = h.ctx.dog_refusals.clone();
let watch = spawn_silent_dog_watch(h.ctx.supervisor.clone(), refusals.clone());
tokio::task::yield_now().await;
let ticks_in_a_budget = (DOG_SILENCE_BUDGET.as_nanos() / DOG_SILENCE_POLL.as_nanos())
.try_into()
.expect("a silence budget of a few seconds fits in a u32 tick count");
for _ in 0..ticks_in_a_budget {
tokio::time::advance(DOG_SILENCE_POLL).await;
tokio::task::yield_now().await;
}
for _ in 0..64 {
if !refusals.restarting().is_empty() {
break;
}
tokio::task::yield_now().await;
}
assert_eq!(
refusals.restarting(),
vec!["metrics".to_string()],
"one budget of silence, driven through the watcher's own tick, must earn exactly one restart"
);
assert!(
refusals.stale().is_empty(),
"one silence is a dog to restart, not a dog to give up on"
);
watch.abort();
}
}