use core::fmt;
use core::time::Duration;
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex, PoisonError};
use shep_core::barks::{self, Bark};
use shep_core::config::{AppConfig, DogsConfig, ResolvedApp, normalize};
use shep_core::paths::ShepPaths;
use shep_core::protocol::{BusEvent, DogSource, ProcessEventKind, ProcessInfo};
use shep_core::selector::ProcessSelector;
use shep_core::status::ProcStatus;
use tokio::sync::broadcast::{self, error::RecvError};
use tokio::time::Instant;
use crate::bus::{Bus, SharedEvent};
use crate::supervisor::SupervisorHandle;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DogSpec {
pub name: String,
pub source: DogSource,
}
#[non_exhaustive]
#[derive(Debug)]
pub enum DogError {
NoBinary(std::io::Error),
UnsupportedSource(String),
Config(String),
Io(std::io::Error),
}
impl fmt::Display for DogError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::NoBinary(err) => write!(f, "this binary's own path is unresolvable: {err}"),
Self::UnsupportedSource(source) => {
write!(f, "no way to spawn a dog from source {source}")
}
Self::Config(msg) => write!(f, "dog configuration is unusable: {msg}"),
Self::Io(err) => write!(f, "dog configuration could not be read: {err}"),
}
}
}
impl core::error::Error for DogError {
fn source(&self) -> Option<&(dyn core::error::Error + 'static)> {
match self {
Self::NoBinary(err) | Self::Io(err) => Some(err),
Self::UnsupportedSource(_) | Self::Config(_) => None,
}
}
}
#[cfg(unix)]
fn builtin_program() -> Result<PathBuf, DogError> {
crate::handover::exec_target().map_err(DogError::NoBinary)
}
#[cfg(windows)]
fn builtin_program() -> Result<PathBuf, DogError> {
std::env::current_exe().map_err(DogError::NoBinary)
}
pub fn dog_app(spec: &DogSpec, paths: &ShepPaths) -> Result<ResolvedApp, DogError> {
let (script, args) = match &spec.source {
DogSource::BuiltIn => (
builtin_program()?.display().to_string(),
vec!["dog".to_string(), spec.name.clone()],
),
DogSource::Adopted { path } => (path.clone(), Vec::new()),
source => return Err(DogError::UnsupportedSource(format!("{source:?}"))),
};
let mut config = AppConfig::minimal(&spec.name, &script);
config.args = args;
config
.env
.insert("SHEP_HOME".to_string(), paths.home.display().to_string());
config
.env
.insert("SHEP_DOG_NAME".to_string(), spec.name.clone());
normalize(config).map_err(|err| DogError::Config(err.to_string()))
}
pub async fn spawn_enabled_dogs(
specs: &[DogSpec],
paths: &ShepPaths,
supervisor: &SupervisorHandle,
events: &Bus,
) {
for spec in specs {
let app = match dog_app(spec, paths) {
Ok(app) => app,
Err(err) => {
tracing::warn!(dog = %spec.name, %err, "a dog did not start");
continue;
}
};
let script = app.config().script.clone();
match supervisor.start_dog(app, spec.source.clone()).await {
Ok(info) if info.dog.is_none() => tracing::warn!(
dog = %spec.name,
"a sheep is already registered under this name; the dog did not start"
),
Ok(info) => {
narrate(
events,
&info,
&format!("shep has this dog enabled, running the binary at {script}"),
)
.await;
}
Err(err) => tracing::warn!(dog = %spec.name, %err, "a dog did not start"),
}
}
}
pub fn dog_section(path: &Path, name: &str) -> Result<String, DogError> {
let source = match std::fs::read_to_string(path) {
Ok(source) => source,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(String::new()),
Err(err) => return Err(DogError::Io(err)),
};
let config =
DogsConfig::load(Some(&source)).map_err(|err| DogError::Config(err.to_string()))?;
let Some(table) = config.dog.get(name) else {
return Ok(String::new());
};
let doc: toml_edit::DocumentMut = source
.parse()
.map_err(|err: toml_edit::TomlError| DogError::Config(err.to_string()))?;
match doc.get(name) {
Some(toml_edit::Item::Table(spanned)) => {
let mut section = spanned.clone();
section.set_implicit(true);
*section.decor_mut() = toml_edit::Decor::default();
let mut out = toml_edit::DocumentMut::new();
*out.as_table_mut() = section;
Ok(out.to_string())
}
_ => toml::to_string(table).map_err(|err| DogError::Config(err.to_string())),
}
}
pub fn set_dog_section(path: &Path, name: &str, section: &str) -> Result<(), DogError> {
use std::io::Write as _;
use toml_edit::{DocumentMut, Item};
let _lock = shep_core::config_lock::ConfigLock::acquire(path).map_err(DogError::Io)?;
let existing = match std::fs::read_to_string(path) {
Ok(existing) => existing,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => String::new(),
Err(err) => return Err(DogError::Io(err)),
};
let mut doc: DocumentMut = existing.parse().map_err(|err: toml_edit::TomlError| {
DogError::Config(format!("dogs.toml does not parse: {err}"))
})?;
let incoming: DocumentMut = section.parse().map_err(|err: toml_edit::TomlError| {
DogError::Config(format!("[{name}] does not parse: {err}"))
})?;
let mut table = incoming.as_table().clone();
table.set_implicit(false);
if let Some(Item::Table(existing)) = doc.get(name) {
*table.decor_mut() = existing.decor().clone();
}
doc[name] = Item::Table(table);
let rendered = doc.to_string();
DogsConfig::load(Some(&rendered))
.map_err(|err| DogError::Config(format!("[{name}] would not load: {err}")))?;
let parent = path.parent().unwrap_or_else(|| Path::new("."));
let mut tmp = shep_core::config_lock::create_config_file(parent).map_err(DogError::Io)?;
tmp.write_all(rendered.as_bytes()).map_err(DogError::Io)?;
shep_core::atomic_file::publish(tmp, path).map_err(DogError::Io)
}
#[non_exhaustive]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Refusal {
Restart,
Stale,
AlreadyStale,
}
#[derive(Debug, Clone, Default)]
pub struct DogRefusals {
seen: Arc<Mutex<Links>>,
}
#[derive(Debug, Default)]
struct Links {
refusals: BTreeMap<String, u32>,
handshook: BTreeSet<String>,
}
impl DogRefusals {
#[must_use]
pub fn new() -> Self {
Self::default()
}
pub fn refused(&self, name: &str) -> Refusal {
let mut seen = self.lock();
seen.handshook.remove(name);
let count = seen.refusals.entry(name.to_string()).or_insert(0);
*count = count.saturating_add(1);
match *count {
1 => Refusal::Restart,
2 => Refusal::Stale,
_ => Refusal::AlreadyStale,
}
}
pub fn handshook(&self, name: &str) -> bool {
let mut seen = self.lock();
seen.refusals.remove(name);
seen.handshook.insert(name.to_string())
}
#[must_use]
pub fn has_handshook(&self, name: &str) -> bool {
self.lock().handshook.contains(name)
}
#[must_use]
pub fn restarting(&self) -> Vec<String> {
self.lock()
.refusals
.iter()
.filter(|(_, count)| **count == 1)
.map(|(name, _)| name.clone())
.collect()
}
#[must_use]
pub fn stale(&self) -> Vec<String> {
self.lock()
.refusals
.iter()
.filter(|(_, count)| **count >= 2)
.map(|(name, _)| name.clone())
.collect()
}
fn lock(&self) -> std::sync::MutexGuard<'_, Links> {
self.seen.lock().unwrap_or_else(PoisonError::into_inner)
}
}
const PEER_CONTACT_CAPACITY: usize = 1024;
const PEER_CONTACT_WARMUP: Duration = DOG_SILENCE_BUDGET;
#[derive(Debug, Clone, Default)]
pub struct PeerContacts {
seen: Arc<Mutex<Contacts>>,
}
#[derive(Debug)]
struct Contacts {
watching_since: Instant,
by_pid: BTreeMap<u32, Seen>,
clock: u64,
}
#[derive(Debug)]
struct Seen {
named_a_dog: bool,
touched: u64,
}
#[non_exhaustive]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Contact {
None,
Anonymous,
Named,
Unknown,
}
impl PeerContacts {
#[must_use]
pub fn new() -> Self {
Self::default()
}
pub fn connected(&self, pid: u32) {
let mut seen = self.lock();
seen.clock = seen.clock.saturating_add(1);
let clock = seen.clock;
match seen.by_pid.get_mut(&pid) {
Some(entry) => entry.touched = clock,
None => {
seen.by_pid.insert(
pid,
Seen {
named_a_dog: false,
touched: clock,
},
);
seen.evict_oldest();
}
}
}
pub fn named_a_dog(&self, pid: u32) {
let mut seen = self.lock();
seen.clock = seen.clock.saturating_add(1);
let clock = seen.clock;
let entry = seen.by_pid.entry(pid).or_insert(Seen {
named_a_dog: false,
touched: clock,
});
entry.named_a_dog = true;
entry.touched = clock;
seen.evict_oldest();
}
#[must_use]
pub fn is_warming(&self) -> bool {
self.lock().watching_since.elapsed() < PEER_CONTACT_WARMUP
}
#[cfg(test)]
pub(crate) fn force_warm(&self) {
let mut seen = self.lock();
seen.watching_since = Instant::now() - PEER_CONTACT_WARMUP * 2;
}
#[must_use]
pub fn from_pid(&self, pid: Option<u32>) -> Contact {
let Some(pid) = pid else {
return Contact::Unknown;
};
let seen = self.lock();
match seen.by_pid.get(&pid) {
None if seen.watching_since.elapsed() < PEER_CONTACT_WARMUP => Contact::Unknown,
None => Contact::None,
Some(seen) if seen.named_a_dog => Contact::Named,
Some(_) => Contact::Anonymous,
}
}
fn lock(&self) -> std::sync::MutexGuard<'_, Contacts> {
self.seen.lock().unwrap_or_else(PoisonError::into_inner)
}
}
impl Default for Contacts {
fn default() -> Self {
Self {
watching_since: Instant::now(),
by_pid: BTreeMap::new(),
clock: 0,
}
}
}
impl Contacts {
fn evict_oldest(&mut self) {
if self.by_pid.len() <= PEER_CONTACT_CAPACITY {
return;
}
let oldest = self
.by_pid
.iter()
.min_by_key(|(_, seen)| seen.touched)
.map(|(pid, _)| *pid);
if let Some(pid) = oldest {
self.by_pid.remove(&pid);
}
}
}
pub fn record_refused_dog(
name: &str,
client_version: &str,
refusals: &DogRefusals,
supervisor: &SupervisorHandle,
) -> Refusal {
let verdict = refusals.refused(name);
match verdict {
Refusal::Restart => {
tracing::warn!(
dog = %name,
dog_version = %client_version,
"refused a dog on protocol skew; restarting it once from the binary on disk"
);
let supervisor = supervisor.clone();
let name = name.to_string();
tokio::spawn(async move { restart_refused_dog(&supervisor, &name).await });
}
Refusal::Stale => tracing::error!(
dog = %name,
dog_version = %client_version,
"refused a dog on protocol skew again after restarting it: the binary on disk speaks the same protocol the running one did, so this dog is stale and will not be restarted again. Rebuild or reinstall it against this shep"
),
Refusal::AlreadyStale => tracing::debug!(
dog = %name,
dog_version = %client_version,
"refused a dog already reported stale"
),
}
verdict
}
async fn restart_refused_dog(supervisor: &SupervisorHandle, name: &str) {
match supervisor
.restart_automatic(ProcessSelector::Name(name.to_string()))
.await
{
Ok(_) => tracing::info!(dog = %name, "restarted a refused dog from the binary on disk"),
Err(err) => tracing::warn!(dog = %name, %err, "a refused dog could not be restarted"),
}
}
pub const DOG_SILENCE_BUDGET: Duration = Duration::from_secs(5);
const DOG_SILENCE_POLL: Duration = Duration::from_secs(1);
pub(crate) fn silent_dogs(infos: &[ProcessInfo], refusals: &DogRefusals) -> Vec<String> {
let stale = refusals.stale();
let mut names: Vec<String> = infos
.iter()
.filter(|info| {
info.dog.is_some()
&& matches!(info.status, ProcStatus::Starting | ProcStatus::Online)
&& !refusals.has_handshook(&info.name)
&& !stale.contains(&info.name)
})
.map(|info| info.name.clone())
.collect();
names.sort();
names.dedup();
names
}
#[derive(Debug, Default)]
pub(crate) struct SilentDogs {
first_seen: BTreeMap<String, Instant>,
}
impl SilentDogs {
fn due(&mut self, silent: &[String], now: Instant) -> Vec<String> {
self.first_seen.retain(|name, _| silent.contains(name));
let mut due = Vec::new();
for name in silent {
let since = self.first_seen.entry(name.clone()).or_insert(now);
if now.saturating_duration_since(*since) >= DOG_SILENCE_BUDGET {
*since = now;
due.push(name.clone());
}
}
due
}
}
pub(crate) async fn check_silent_dogs(
supervisor: &SupervisorHandle,
refusals: &DogRefusals,
contacts: &PeerContacts,
events: &Bus,
seen: &mut SilentDogs,
now: Instant,
) -> Vec<(String, Refusal)> {
if contacts.is_warming() {
return Vec::new();
}
let Ok(infos) = supervisor.list_checked().await else {
return Vec::new();
};
let silent = silent_dogs(&infos, refusals);
let mut acted = Vec::new();
for name in seen.due(&silent, now) {
let info = infos.iter().find(|info| info.name == name);
let evidence = Silence::of(info.and_then(|info| info.pid), contacts);
let verdict = record_silent_dog(&name, info, evidence, refusals, events, supervisor).await;
acted.push((name, verdict));
}
acted
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Silence {
Unreachable {
pid: u32,
},
Anonymous {
pid: u32,
},
Unattributed,
}
impl Silence {
fn of(pid: Option<u32>, contacts: &PeerContacts) -> Self {
match (pid, contacts.from_pid(pid)) {
(Some(pid), Contact::None) => Self::Unreachable { pid },
(Some(pid), Contact::Anonymous) => Self::Anonymous { pid },
_ => Self::Unattributed,
}
}
}
async fn record_silent_dog(
name: &str,
info: Option<&ProcessInfo>,
evidence: Silence,
refusals: &DogRefusals,
events: &Bus,
supervisor: &SupervisorHandle,
) -> Refusal {
let verdict = refusals.refused(name);
match verdict {
Refusal::Restart => {
let seen = first_rung_evidence(evidence);
tracing::warn!(
dog = %name,
silent_for_secs = DOG_SILENCE_BUDGET.as_secs(),
evidence = %seen,
"a dog has been running without ever answering this shepherd; restarting it once from the binary on disk"
);
if let Some(info) = info {
narrate(
events,
info,
&format!(
"this dog has been running for {}s without ever answering this shepherd: {seen}. Restarting it once from the binary on disk",
DOG_SILENCE_BUDGET.as_secs()
),
)
.await;
}
restart_refused_dog(supervisor, name).await;
}
Refusal::Stale => {
let verdict = stale_verdict(name, evidence);
tracing::error!(dog = %name, "{verdict}");
if let Some(info) = info {
narrate(events, info, &verdict).await;
}
}
Refusal::AlreadyStale => tracing::debug!(
dog = %name,
"a silent dog that was already reported stale"
),
}
verdict
}
fn first_rung_evidence(evidence: Silence) -> String {
match evidence {
Silence::Unreachable { pid } => {
format!("nothing has connected to this shepherd from pid {pid}")
}
Silence::Anonymous { pid } => format!(
"pid {pid} has connected to this shepherd without naming a dog, so the restart is unlikely to help"
),
Silence::Unattributed => {
"this shepherd cannot tell which process opened a connection".to_string()
}
}
}
fn stale_verdict(name: &str, evidence: Silence) -> String {
let seen = "a dog restarted for never answering this shepherd has still not answered it";
match evidence {
Silence::Unreachable { pid } => format!(
"{seen}, and nothing has ever connected to this shepherd's socket from its process (pid {pid}): \
the binary on disk cannot reach this shep either, so this dog is stale and will not be \
restarted again. Read its own log with `shep bleats {name}` for what it says about \
connecting, then rebuild or reinstall it and run `shep restart {name}`. A dog \
installed with cargo wants `cargo install <crate> --force`: its own version does \
not change when the shep it was built against does, so a plain `cargo install` \
reports the package already installed, builds nothing, and exits 0"
),
Silence::Anonymous { pid } => format!(
"{seen}, but its process (pid {pid}) HAS connected to this shepherd — every time without \
naming a dog in its handshake, which is the only thing this shepherd waits for. The dog \
is reaching shep and may be serving every request it is asked; reinstalling the same \
build will NOT change that. It is built against shep-client older than 0.1.23, or it \
connects with `Client::connect` instead of `ReconnectingClient::connect_as_dog`. Rebuild \
it against shep-client 0.1.23 or newer, then run `shep restart {name}`. With cargo \
that means `cargo install <crate> --force`: the dog's own version does not change \
when its shep-client does, so a plain `cargo install` builds nothing and exits 0. \
It will not be restarted again in the meantime, and it goes on running"
),
Silence::Unattributed => format!(
"{seen}, and this shepherd could not tell which process opened its connections, so it \
cannot say which of two things is wrong. Either the dog is not reaching the socket at \
all — rebuild or reinstall it — or it is reaching it and never names itself in the \
handshake, which means a build against shep-client older than 0.1.23 and which \
reinstalling the same build will not fix. Run `shep bleats {name}` to tell them apart: \
a dog that cannot reach the socket says so in its own log, and one that is connected \
and merely anonymous does not. It will not be restarted again"
),
}
}
pub fn spawn_silent_dog_watch(
supervisor: SupervisorHandle,
refusals: DogRefusals,
contacts: PeerContacts,
events: Bus,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
let mut ticks = tokio::time::interval(DOG_SILENCE_POLL);
ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
let mut seen = SilentDogs::default();
loop {
let now = ticks.tick().await;
check_silent_dogs(&supervisor, &refusals, &contacts, &events, &mut seen, now).await;
}
})
}
const SHEP_VOICE: &str = "[shep]";
pub(crate) async fn narrate(events: &Bus, info: &ProcessInfo, message: &str) {
let line = format!("{SHEP_VOICE} {message}");
if let Some(path) = &info.err_file {
let mut written = String::with_capacity(line.len() + 32);
shep_core::logstamp::stamp_into(&mut written);
written.push_str(&line);
written.push('\n');
if let Ok(mut file) = crate::tokio_runner::open_append(Path::new(path)).await {
use tokio::io::AsyncWriteExt as _;
let _record = crate::tokio_runner::record_lock(Path::new(path))
.lock_owned()
.await;
let written = async {
file.write_all(written.as_bytes()).await?;
file.flush().await
}
.await;
if let Err(error) = written {
tracing::warn!(
dog = %info.name,
%error,
"shep's own narration did not reach this dog's log"
);
}
}
}
events.publish_log(BusEvent::LogErr { id: info.id, line });
}
pub(crate) fn narrate_by_name(
supervisor: &SupervisorHandle,
events: &Bus,
name: &str,
message: String,
) {
let supervisor = supervisor.clone();
let events = events.clone();
let name = name.to_string();
tokio::spawn(async move {
let Ok(infos) = supervisor.list_checked().await else {
return;
};
if let Some(info) = infos
.iter()
.find(|info| info.name == name && info.dog.is_some())
{
narrate(&events, info, &message).await;
}
});
}
fn exit_words(info: &ProcessInfo) -> String {
match info.last_exit {
Some(exit) => match (exit.code, exit.signal) {
(Some(code), _) => format!("this dog's process exited with code {code}"),
(None, Some(signal)) => {
format!("this dog's process was killed by signal {signal}")
}
(None, None) => {
"this dog's process stopped, and the OS reported neither an exit code nor a signal"
.to_string()
}
},
None => "this dog's process stopped, and this shepherd has no record of how".to_string(),
}
}
pub fn spawn_dog_watch(
mut events: broadcast::Receiver<SharedEvent>,
publish: Bus,
barks: PathBuf,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
loop {
match events.recv().await {
Ok(event) => {
let BusEvent::Process {
event: kind, info, ..
} = &*event
else {
continue;
};
if info.dog.is_none() {
continue;
}
match kind {
ProcessEventKind::Errored => {
record_dog_errored(&barks, &info.name, info.restarts);
}
ProcessEventKind::Start => {
let pid = info
.pid
.map_or_else(|| "unknown".to_string(), |pid| pid.to_string());
narrate(
&publish,
info,
&format!("shep started this dog; its process is pid {pid}"),
)
.await;
}
ProcessEventKind::Exit => {
narrate(&publish, info, &exit_words(info)).await;
}
_ => {}
}
}
Err(RecvError::Lagged(count)) => {
tracing::warn!(
count,
"the shepherd's dog watch dropped bus events; a dog's exhausted restart budget may have gone unrecorded"
);
}
Err(RecvError::Closed) => break,
}
}
})
}
fn record_dog_errored(barks_path: &Path, name: &str, restarts: u32) {
let budget = AppConfig::default().max_restarts;
tracing::error!(dog = %name, restarts, budget, "a dog exhausted its restart budget");
let bark = Bark {
at_ms: crate::now_ms(),
rule: "daemon".to_string(),
subject: name.to_string(),
message: format!(
"dog {name} exhausted its restart budget: {restarts} restarts against a budget of {budget}"
),
sinks: Vec::new(),
};
if let Err(err) = barks::append(barks_path, &bark, barks::DEFAULT_MAX_BYTES) {
tracing::warn!(%err, dog = %name, "failed to record a dog's exhausted restart budget");
}
}
#[cfg(test)]
mod tests {
#[test]
fn a_section_written_with_sub_tables_reaches_the_dog_as_its_own_document() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("dogs.toml");
let source = "# above the header\n[bark]\n# above poll\npoll = \"5s\"\n\n[bark.sinks.local]\nkind = \"json\"\nurl = \"https://example.invalid/h\"\n\n[[bark.rules]]\non = \"event\"\nkinds = [\"exit\"]\nsinks = [\"local\"]\n\n[otel]\nendpoint = \"https://collector.invalid\"\n";
std::fs::write(&path, source).unwrap();
let section = dog_section(&path, "bark").unwrap();
let parsed: toml::Table = section.parse().unwrap();
assert!(parsed.contains_key("sinks"), "{section}");
assert!(parsed.contains_key("rules"), "{section}");
assert_eq!(
parsed["sinks"]["local"]["kind"].as_str(),
Some("json"),
"{section}"
);
assert_eq!(
parsed["rules"].as_array().map(Vec::len),
Some(1),
"{section}"
);
assert!(section.contains("# above poll"), "{section}");
assert!(!section.contains("# above the header"), "{section}");
assert!(!section.contains("otel"), "{section}");
}
#[test]
fn a_sub_table_section_survives_a_read_and_a_write_back() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("dogs.toml");
std::fs::write(
&path,
"# keep me\n[bark]\npoll = \"5s\"\n\n[bark.sinks.local]\nkind = \"json\"\nurl = \"https://example.invalid/h\"\n",
)
.unwrap();
let section = dog_section(&path, "bark").unwrap();
set_dog_section(&path, "bark", §ion).unwrap();
let after = std::fs::read_to_string(&path).unwrap();
let reloaded: toml::Table = after.parse().unwrap();
assert_eq!(
reloaded["bark"]["sinks"]["local"]["url"].as_str(),
Some("https://example.invalid/h"),
"{after}"
);
assert!(after.contains("# keep me"), "{after}");
assert_eq!(dog_section(&path, "bark").unwrap(), section, "{after}");
}
use super::*;
use crate::fake::ProcScript;
use crate::testing::test_paths;
use shep_core::protocol::ProcessInfo;
use shep_core::status::ProcStatus;
#[cfg(unix)]
#[test]
fn a_deleted_inode_answer_from_current_exe_never_becomes_a_dogs_script() {
let refusal = crate::handover::resolve_target(
[None, Some(PathBuf::from("/opt/shep/shep (deleted)"))],
None,
)
.unwrap_err();
let err = DogError::NoBinary(refusal);
assert_eq!(
err.to_string(),
"this binary's own path is unresolvable: no binary to exec: \
/opt/shep/shep (deleted) (names a deleted inode, not a file)"
);
}
#[test]
fn a_dogs_child_environment_carries_shep_home_and_its_name_and_no_configuration() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
std::fs::write(
&paths.dogs_config,
"[bark]\nwebhook = \"https://example.invalid/hook\"\n",
)
.unwrap();
let spec = DogSpec {
name: "bark".to_string(),
source: DogSource::BuiltIn,
};
let app = dog_app(&spec, &paths).unwrap();
let assembled = crate::assemble::assemble(
&app,
0,
&paths,
None,
&shep_core::secrets::SecretView::empty("production".to_string()),
)
.expect("a dog's own config carries no template to refuse");
assert_eq!(
assembled.env.get("SHEP_HOME"),
Some(&paths.home.display().to_string())
);
assert_eq!(
assembled.env.get("SHEP_DOG_NAME"),
Some(&"bark".to_string()),
"a dog is told the name its own section lives under"
);
assert!(
!assembled
.env
.values()
.any(|v| v.contains("example.invalid")),
"a dog's configuration never travels in its environment: {:?}",
assembled.env
);
}
#[test]
fn a_built_in_dog_runs_this_binary_and_an_adopted_one_runs_its_own() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let built_in = dog_app(
&DogSpec {
name: "metrics".to_string(),
source: DogSource::BuiltIn,
},
&paths,
)
.unwrap();
assert_eq!(
built_in.config().script,
std::env::current_exe().unwrap().display().to_string()
);
assert_eq!(built_in.config().args, vec!["dog", "metrics"]);
let adopted = dog_app(
&DogSpec {
name: "otel".to_string(),
source: DogSource::Adopted {
path: "/usr/local/bin/shep-otel".to_string(),
},
},
&paths,
)
.unwrap();
assert_eq!(adopted.config().script, "/usr/local/bin/shep-otel");
assert!(adopted.config().args.is_empty());
assert_eq!(
adopted.config().name,
"otel",
"the NAME is the config key, never the filename"
);
}
#[test]
fn an_adopted_dog_is_told_the_name_it_was_registered_under() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let adopted = dog_app(
&DogSpec {
name: "telemetry".to_string(),
source: DogSource::Adopted {
path: "/usr/local/bin/shep-otel".to_string(),
},
},
&paths,
)
.unwrap();
assert!(
adopted.config().args.is_empty(),
"the name arrives without shep inventing an argv for a foreign binary"
);
assert_eq!(
adopted.config().env.get("SHEP_DOG_NAME"),
Some(&"telemetry".to_string())
);
}
#[test]
fn a_dogs_section_comes_back_as_its_own_table_and_nothing_else() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("dogs.toml");
std::fs::write(
&path,
"[bark]\ndebounce = \"30s\"\n\n[metrics]\nport = 9615\n",
)
.unwrap();
let bark = dog_section(&path, "bark").unwrap();
assert!(bark.contains("debounce"));
assert!(
!bark.contains("9615"),
"one dog never sees another's config"
);
let parsed: toml::Table = toml::from_str(&bark).unwrap();
assert_eq!(parsed["debounce"].as_str(), Some("30s"));
assert_eq!(dog_section(&path, "absent").unwrap(), "");
assert_eq!(
dog_section(&dir.path().join("gone.toml"), "bark").unwrap(),
""
);
}
#[test]
fn a_section_reaches_the_wire_exactly_as_it_did_from_shep_toml() {
let dir = tempfile::tempdir().expect("tempdir");
let path = dir.path().join("dogs.toml");
std::fs::write(&path, "[bark]\ndebounce = \"30s\"\n").expect("write");
assert_eq!(
dog_section(&path, "bark").expect("section"),
"debounce = \"30s\"\n"
);
}
#[test]
fn a_dog_with_no_section_still_gets_an_empty_string() {
let dir = tempfile::tempdir().expect("tempdir");
let path = dir.path().join("dogs.toml");
std::fs::write(&path, "[bark]\ndebounce = \"30s\"\n").expect("write");
assert_eq!(dog_section(&path, "metrics").expect("section"), "");
}
#[test]
fn set_dog_section_replaces_one_table_and_leaves_the_rest_untouched() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("dogs.toml");
std::fs::write(
&path,
"# top comment\n[metrics]\nbind = \"127.0.0.1:9100\"\n\n[bark]\npoll = \"60s\"\n",
)
.unwrap();
set_dog_section(&path, "bark", "poll = \"30s\"\nhistory_bytes = 4096\n").unwrap();
let text = std::fs::read_to_string(&path).unwrap();
assert!(text.starts_with("# top comment"), "{text}");
assert!(text.contains("bind = \"127.0.0.1:9100\""), "{text}");
assert!(text.contains("poll = \"30s\""), "{text}");
assert!(!text.contains("poll = \"60s\""), "{text}");
let parsed = DogsConfig::load(Some(&text)).unwrap();
assert_eq!(parsed.dog["bark"]["history_bytes"].as_integer(), Some(4096));
}
#[test]
fn a_pane_round_trip_keeps_the_comments_and_the_key_order() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("dogs.toml");
std::fs::write(
&path,
"# what bark is for\n[bark]\n# why bark polls slowly\npoll = \"60s\"\nzz_last = 1\naa_first = 2\n",
)
.unwrap();
let section = dog_section(&path, "bark").unwrap();
assert!(section.contains("# why bark polls slowly"), "{section}");
assert!(
section.find("zz_last") < section.find("aa_first"),
"the keys come back in the operator\'s order: {section}"
);
set_dog_section(&path, "bark", §ion.replace("60s", "30s")).unwrap();
let text = std::fs::read_to_string(&path).unwrap();
assert!(text.contains("# why bark polls slowly"), "{text}");
assert!(
text.contains("# what bark is for"),
"the header's own comment lives on the table, not inside it: {text}"
);
assert!(text.contains("poll = \"30s\""), "{text}");
assert!(
text.find("zz_last") < text.find("aa_first"),
"the write keeps the order the read handed it: {text}"
);
}
#[test]
fn set_dog_section_creates_the_file_when_there_is_none() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("dogs.toml");
set_dog_section(&path, "bark", "poll = \"30s\"\n").unwrap();
let parsed = DogsConfig::load(Some(&std::fs::read_to_string(&path).unwrap())).unwrap();
assert!(parsed.dog.contains_key("bark"));
}
#[test]
fn set_dog_section_refuses_text_that_is_not_a_table_and_writes_nothing() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("dogs.toml");
std::fs::write(&path, "[bark]\npoll = \"60s\"\n").unwrap();
let err = set_dog_section(&path, "bark", "this is = = not toml").unwrap_err();
assert!(err.to_string().contains("bark"), "{err}");
assert_eq!(
std::fs::read_to_string(&path).unwrap(),
"[bark]\npoll = \"60s\"\n"
);
}
#[test]
fn set_dog_section_refuses_a_result_the_daemon_could_not_read_back() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("dogs.toml");
std::fs::write(&path, "port = 9100\n[bark]\npoll = \"60s\"\n").unwrap();
let err = set_dog_section(&path, "bark", "poll = \"30s\"\n").unwrap_err();
assert!(err.to_string().contains("bark"), "{err}");
assert_eq!(
std::fs::read_to_string(&path).unwrap(),
"port = 9100\n[bark]\npoll = \"60s\"\n"
);
}
#[test]
fn set_dog_section_leaves_the_table_behind_when_the_section_is_emptied() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("dogs.toml");
std::fs::write(&path, "[bark]\npoll = \"60s\"\n").unwrap();
set_dog_section(&path, "bark", "").unwrap();
let text = std::fs::read_to_string(&path).unwrap();
assert!(text.contains("[bark]"), "{text}");
let parsed = DogsConfig::load(Some(&text)).unwrap();
assert!(parsed.dog["bark"].is_empty(), "{text}");
}
#[cfg(unix)]
#[test]
fn set_dog_section_writes_owner_only() {
use std::os::unix::fs::PermissionsExt as _;
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("dogs.toml");
set_dog_section(&path, "bark", "poll = \"30s\"\n").unwrap();
assert_eq!(
std::fs::metadata(&path).unwrap().permissions().mode() & 0o777,
0o600
);
}
fn process_event(name: &str, kind: ProcessEventKind, dog: Option<DogSource>) -> SharedEvent {
SharedEvent::new(BusEvent::Process {
event: kind,
info: ProcessInfo::builder(1, name, ProcStatus::Errored)
.restarts(16)
.dog(dog)
.build(),
manually: false,
at_ms: 1_700_000_000_000,
})
}
fn errored_event(name: &str, dog: Option<DogSource>) -> SharedEvent {
process_event(name, ProcessEventKind::Errored, dog)
}
async fn await_barks(path: &std::path::Path, n: usize) -> Vec<Bark> {
tokio::time::timeout(std::time::Duration::from_secs(2), async {
loop {
let found = barks::read(path).unwrap_or_default();
if found.len() >= n {
return found;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
})
.await
.expect("barks.jsonl never reached the expected record count")
}
#[tokio::test]
async fn the_shepherd_records_a_dog_that_gave_up_and_leaves_the_sheep_to_bark() {
let dir = tempfile::tempdir().unwrap();
let barks_path = dir.path().join("barks.jsonl");
let (events, rx) = crate::bus::test_bus(16);
let watch = spawn_dog_watch(rx, events.clone(), barks_path.clone());
events.send(errored_event("web", None)).unwrap();
events
.send(errored_event("bark", Some(DogSource::BuiltIn)))
.unwrap();
let recorded = await_barks(&barks_path, 1).await;
assert_eq!(recorded.len(), 1, "one record, and it is the dog's");
assert_eq!(recorded[0].subject, "bark");
assert_eq!(recorded[0].rule, "daemon");
assert!(
recorded[0].sinks.is_empty(),
"the shepherd has no sinks and says so by carrying none"
);
watch.abort();
}
#[tokio::test]
async fn a_dog_that_merely_exited_is_not_recorded() {
let dir = tempfile::tempdir().unwrap();
let barks_path = dir.path().join("barks.jsonl");
let (events, rx) = crate::bus::test_bus(16);
let watch = spawn_dog_watch(rx, events.clone(), barks_path.clone());
events
.send(process_event(
"bark",
ProcessEventKind::Exit,
Some(DogSource::BuiltIn),
))
.unwrap();
events
.send(errored_event("bark", Some(DogSource::BuiltIn)))
.unwrap();
let recorded = await_barks(&barks_path, 1).await;
assert_eq!(
recorded.len(),
1,
"the Exit left no record; only the Errored that followed it did"
);
watch.abort();
}
#[test]
fn a_refused_dog_earns_one_restart_and_is_then_stale_forever() {
let refusals = DogRefusals::new();
assert!(refusals.stale().is_empty());
assert_eq!(refusals.refused("metrics"), Refusal::Restart);
assert!(
refusals.stale().is_empty(),
"one refusal is a dog to restart, not a dog to give up on"
);
assert_eq!(refusals.refused("metrics"), Refusal::Stale);
assert_eq!(refusals.stale(), vec!["metrics".to_string()]);
for _ in 0..5 {
assert_eq!(refusals.refused("metrics"), Refusal::AlreadyStale);
}
assert_eq!(refusals.stale(), vec!["metrics".to_string()]);
}
#[test]
fn a_dog_that_gets_in_is_owed_a_fresh_restart_if_it_is_ever_refused_again() {
let refusals = DogRefusals::new();
assert_eq!(refusals.refused("metrics"), Refusal::Restart);
refusals.handshook("metrics");
assert!(refusals.stale().is_empty());
assert_eq!(
refusals.refused("metrics"),
Refusal::Restart,
"the restart that fixed it must not be charged against the next episode"
);
}
#[test]
fn each_dog_carries_its_own_count() {
let refusals = DogRefusals::new();
assert_eq!(refusals.refused("bark"), Refusal::Restart);
assert_eq!(refusals.refused("bark"), Refusal::Stale);
assert_eq!(
refusals.refused("metrics"),
Refusal::Restart,
"bark's two refusals are bark's"
);
refusals.handshook("metrics");
assert_eq!(
refusals.stale(),
vec!["bark".to_string()],
"one dog getting in says nothing about another"
);
}
#[test]
fn a_dog_mid_restart_is_neither_stale_nor_settled() {
let refusals = DogRefusals::new();
assert!(refusals.restarting().is_empty());
refusals.refused("metrics");
assert_eq!(refusals.restarting(), vec!["metrics".to_string()]);
assert!(refusals.stale().is_empty());
refusals.refused("metrics");
assert!(
refusals.restarting().is_empty(),
"a dog that has been given up on is settled, not still being restarted"
);
assert_eq!(refusals.stale(), vec!["metrics".to_string()]);
}
#[test]
fn only_an_accepted_handshake_says_a_dog_has_answered() {
let refusals = DogRefusals::new();
assert!(
!refusals.has_handshook("metrics"),
"a dog nobody has heard from has not answered"
);
refusals.handshook("metrics");
assert!(refusals.has_handshook("metrics"));
assert!(!refusals.has_handshook("bark"), "one dog answers for one");
refusals.refused("metrics");
assert!(
!refusals.has_handshook("metrics"),
"the handshake that earned the mark is the one that just died"
);
}
const DOG_FIXTURE_START_BUDGET: Duration = Duration::from_secs(10);
const SETTLE_STEP: Duration = Duration::from_millis(250);
const LADDER_BUDGET: Duration = Duration::from_secs(45);
async fn settle_until(
what: &str,
within: Duration,
mut settled: impl FnMut() -> bool,
) -> Duration {
let began = Instant::now();
tokio::time::timeout(within, async {
while !settled() {
tokio::time::sleep(SETTLE_STEP).await;
}
})
.await
.unwrap_or_else(|_| panic!("{what} did not happen within {within:?} of virtual time"));
began.elapsed()
}
async fn start_test_dog(ctx: &crate::rpc::RpcContext, name: &str) {
let spec = DogSpec {
name: name.to_string(),
source: DogSource::BuiltIn,
};
let app = dog_app(&spec, &ctx.paths).expect("the dog fixture must assemble");
tokio::time::timeout(
DOG_FIXTURE_START_BUDGET,
ctx.supervisor.start_dog(app, DogSource::BuiltIn),
)
.await
.expect("the dog fixture must start inside its budget")
.expect("the dog fixture must start");
}
#[tokio::test(start_paused = true)]
async fn a_dog_that_never_answers_is_restarted_once_and_then_marked_stale() {
let h = crate::testing::harness(vec![
ProcScript::never_exits(),
ProcScript::never_exits(),
ProcScript::never_exits(),
ProcScript::never_exits(),
]);
start_test_dog(&h.ctx, "metrics").await;
tokio::time::advance(PEER_CONTACT_WARMUP * 2).await;
let refusals = &h.ctx.dog_refusals;
let contacts = &h.ctx.peer_contacts;
let events = &h.ctx.events;
let mut seen = SilentDogs::default();
let t0 = Instant::now();
assert!(
check_silent_dogs(&h.ctx.supervisor, refusals, contacts, events, &mut seen, t0)
.await
.is_empty(),
"a dog seen quiet for the first time has not yet been quiet for any length of time"
);
assert_eq!(
check_silent_dogs(
&h.ctx.supervisor,
refusals,
contacts,
events,
&mut seen,
t0 + DOG_SILENCE_BUDGET
)
.await,
vec![("metrics".to_string(), Refusal::Restart)],
"a whole budget of silence buys the one restart from disk"
);
assert_eq!(refusals.restarting(), vec!["metrics".to_string()]);
assert!(
refusals.stale().is_empty(),
"one silence is a dog to restart, not a dog to give up on"
);
assert_eq!(
check_silent_dogs(
&h.ctx.supervisor,
refusals,
contacts,
events,
&mut seen,
t0 + 2 * DOG_SILENCE_BUDGET
)
.await,
vec![("metrics".to_string(), Refusal::Stale)],
"the restart ran and the dog still has not spoken, so the ladder ends here"
);
assert_eq!(refusals.stale(), vec!["metrics".to_string()]);
assert!(
check_silent_dogs(
&h.ctx.supervisor,
refusals,
contacts,
events,
&mut seen,
t0 + 3 * DOG_SILENCE_BUDGET
)
.await
.is_empty(),
"a dog already given up on is not laddered again, however long it stays quiet"
);
}
#[tokio::test(start_paused = true)]
async fn a_dog_that_answers_inside_the_budget_is_never_touched() {
let h = crate::testing::harness(vec![ProcScript::never_exits()]);
start_test_dog(&h.ctx, "metrics").await;
let refusals = &h.ctx.dog_refusals;
let contacts = &h.ctx.peer_contacts;
let events = &h.ctx.events;
refusals.handshook("metrics");
let mut seen = SilentDogs::default();
let t0 = Instant::now();
for elapsed in [0, 1, 2, 10] {
assert!(
check_silent_dogs(
&h.ctx.supervisor,
refusals,
contacts,
events,
&mut seen,
t0 + elapsed * DOG_SILENCE_BUDGET
)
.await
.is_empty(),
"a dog this shepherd has heard from is not silent at any point on the clock"
);
}
assert!(refusals.restarting().is_empty());
assert!(refusals.stale().is_empty());
}
#[tokio::test(start_paused = true)]
async fn a_dog_already_stale_is_not_laddered_again() {
let h = crate::testing::harness(vec![ProcScript::never_exits()]);
start_test_dog(&h.ctx, "metrics").await;
let refusals = &h.ctx.dog_refusals;
let contacts = &h.ctx.peer_contacts;
let events = &h.ctx.events;
refusals.refused("metrics");
refusals.refused("metrics");
assert_eq!(refusals.stale(), vec!["metrics".to_string()]);
let mut seen = SilentDogs::default();
let t0 = Instant::now();
for elapsed in [0, 1, 2, 5] {
assert!(
check_silent_dogs(
&h.ctx.supervisor,
refusals,
contacts,
events,
&mut seen,
t0 + elapsed * DOG_SILENCE_BUDGET
)
.await
.is_empty(),
"the ladder ends at stale; there is no rung after it to reach"
);
}
}
#[tokio::test(start_paused = true)]
async fn asking_repeatedly_does_not_advance_the_ladder() {
let h = crate::testing::harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
start_test_dog(&h.ctx, "metrics").await;
tokio::time::advance(PEER_CONTACT_WARMUP * 2).await;
let refusals = &h.ctx.dog_refusals;
let contacts = &h.ctx.peer_contacts;
let events = &h.ctx.events;
let mut seen = SilentDogs::default();
let t0 = Instant::now();
for look in 0..20 {
assert!(
check_silent_dogs(
&h.ctx.supervisor,
refusals,
contacts,
events,
&mut seen,
t0 + (DOG_SILENCE_BUDGET / 20) * look
)
.await
.is_empty(),
"look {look} fell inside the budget and must not have moved the dog along"
);
}
assert!(refusals.restarting().is_empty());
assert_eq!(
check_silent_dogs(
&h.ctx.supervisor,
refusals,
contacts,
events,
&mut seen,
t0 + DOG_SILENCE_BUDGET
)
.await,
vec![("metrics".to_string(), Refusal::Restart)],
"the clock is what moves the dog along, and it has now moved"
);
}
#[tokio::test(start_paused = true)]
async fn a_dog_that_never_calls_still_earns_its_rebuild_after_the_warm_up() {
let map_started_no_earlier_than = Instant::now();
let h = crate::testing::harness(vec![
ProcScript::never_exits(),
ProcScript::never_exits(),
ProcScript::never_exits(),
]);
start_test_dog(&h.ctx, "metrics").await;
let refusals = h.ctx.dog_refusals.clone();
let contacts = h.ctx.peer_contacts.clone();
assert!(contacts.is_warming(), "a fresh map starts cold");
let watch = spawn_silent_dog_watch(
h.ctx.supervisor.clone(),
refusals.clone(),
contacts.clone(),
h.ctx.events.clone(),
);
let restart_rung = settle_until("the silent dog's restart rung", LADDER_BUDGET, || {
!refusals.restarting().is_empty()
})
.await;
assert_eq!(
refusals.restarting(),
vec!["metrics".to_string()],
"the dog nothing ever connected from is the one that earns the rung"
);
let first_rung_at = map_started_no_earlier_than.elapsed();
assert!(
first_rung_at >= PEER_CONTACT_WARMUP + DOG_SILENCE_BUDGET,
"a cold map must judge nothing: the first rung landed {first_rung_at:?} in, \
which is inside the {PEER_CONTACT_WARMUP:?} warm-up plus one \
{DOG_SILENCE_BUDGET:?} budget of silence it has to wait out"
);
assert!(
restart_rung >= DOG_SILENCE_BUDGET,
"no rung can be earned in less than a whole budget of silence: {restart_rung:?}"
);
settle_until("the silent dog's stale rung", LADDER_BUDGET, || {
refusals.stale().contains(&"metrics".to_string())
})
.await;
let info = h
.ctx
.supervisor
.list()
.await
.into_iter()
.find(|info| info.name == "metrics")
.expect("the dog fixture is listed");
let verdict = stale_verdict("metrics", Silence::of(info.pid, &contacts));
assert!(
verdict.contains("cannot reach this shep"),
"the earned rebuild advice must survive the warm-up: {verdict}"
);
watch.abort();
}
#[tokio::test(start_paused = true)]
async fn the_watcher_restarts_a_silent_dog_after_one_budget_of_paused_time() {
let h = crate::testing::harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
start_test_dog(&h.ctx, "metrics").await;
tokio::time::advance(PEER_CONTACT_WARMUP * 2).await;
let refusals = h.ctx.dog_refusals.clone();
let watch = spawn_silent_dog_watch(
h.ctx.supervisor.clone(),
refusals.clone(),
h.ctx.peer_contacts.clone(),
h.ctx.events.clone(),
);
let waited = settle_until("the silent dog's restart", LADDER_BUDGET, || {
!refusals.restarting().is_empty()
})
.await;
assert_eq!(
refusals.restarting(),
vec!["metrics".to_string()],
"one budget of silence, driven through the watcher's own tick, must earn exactly one restart"
);
assert!(
waited >= DOG_SILENCE_BUDGET,
"a restart is earned by a whole budget of silence, not by less: {waited:?}"
);
assert!(
refusals.stale().is_empty(),
"one silence is a dog to restart, not a dog to give up on"
);
watch.abort();
}
#[tokio::test(start_paused = true)]
async fn a_pid_that_never_called_is_told_apart_from_one_that_called_anonymously() {
let contacts = PeerContacts::new();
tokio::time::advance(PEER_CONTACT_WARMUP * 2).await;
assert_eq!(
contacts.from_pid(Some(4242)),
Contact::None,
"nothing has connected from this pid, and that is a finding"
);
assert_eq!(
contacts.from_pid(None),
Contact::Unknown,
"no pid to ask about is not the same as a pid nothing came from"
);
contacts.connected(4242);
assert_eq!(
contacts.from_pid(Some(4242)),
Contact::Anonymous,
"a connection that named no dog is exactly the case the operator lost two days to"
);
contacts.named_a_dog(4242);
assert_eq!(contacts.from_pid(Some(4242)), Contact::Named);
}
#[tokio::test(start_paused = true)]
async fn a_cold_map_does_not_claim_a_pid_never_called() {
let contacts = PeerContacts::new();
assert_eq!(
contacts.from_pid(Some(4242)),
Contact::Unknown,
"a map this new was not listening long enough for an absence to mean anything"
);
assert_eq!(
stale_verdict("metrics", Silence::of(Some(4242), &contacts)),
stale_verdict("metrics", Silence::Unattributed),
"an unwarmed map must reach the arm that names both candidates"
);
tokio::time::advance(PEER_CONTACT_WARMUP - Duration::from_millis(1)).await;
assert_eq!(contacts.from_pid(Some(4242)), Contact::Unknown);
tokio::time::advance(Duration::from_millis(2)).await;
assert_eq!(
contacts.from_pid(Some(4242)),
Contact::None,
"shep was listening for a whole budget past the dog's silence"
);
assert!(
stale_verdict("metrics", Silence::of(Some(4242), &contacts))
.contains("cannot reach this shep"),
"the earned reinstall advice must survive"
);
}
#[test]
fn a_pid_that_has_named_a_dog_goes_on_having_named_one() {
let contacts = PeerContacts::new();
contacts.named_a_dog(7);
contacts.connected(7);
assert_eq!(contacts.from_pid(Some(7)), Contact::Named);
}
#[tokio::test(start_paused = true)]
async fn a_full_map_forgets_the_pid_that_stopped_calling() {
let contacts = PeerContacts::new();
tokio::time::advance(PEER_CONTACT_WARMUP * 2).await;
let dog = 1;
contacts.named_a_dog(dog);
for pid in 2..=u32::try_from(PEER_CONTACT_CAPACITY).unwrap() {
contacts.connected(pid);
if pid % 8 == 0 {
contacts.connected(dog);
}
}
let stranger = 2;
for pid in 1_000_000..1_000_100 {
contacts.connected(pid);
}
assert_eq!(
contacts.from_pid(Some(dog)),
Contact::Named,
"a peer that keeps calling must outlive a hundred that called once"
);
assert_eq!(
contacts.from_pid(Some(stranger)),
Contact::None,
"the oldest untouched entry is the one the bound spends"
);
assert!(
contacts.lock().by_pid.len() <= PEER_CONTACT_CAPACITY,
"the map must not grow past its bound"
);
}
#[test]
fn the_stale_verdict_claims_only_what_this_shepherd_watched() {
let unreachable = stale_verdict("metrics", Silence::Unreachable { pid: 900 });
assert!(
unreachable.contains("nothing has ever connected"),
"the reinstall advice has to be earned by an observation: {unreachable}"
);
assert!(unreachable.contains("pid 900"), "{unreachable}");
assert!(
unreachable.contains("rebuild or reinstall it"),
"a dog that never reached the socket is the case reinstalling does fix: {unreachable}"
);
let anonymous = stale_verdict("log-rotate", Silence::Anonymous { pid: 901 });
assert!(
!anonymous.contains("cannot reach this shep"),
"this dog reached shep; claiming otherwise is the whole defect: {anonymous}"
);
assert!(
anonymous.contains("reinstalling the same build will NOT"),
"the two days were spent on advice this line has to refuse: {anonymous}"
);
assert!(
anonymous.contains("0.1.23"),
"the fix is a newer shep-client, and the message has to name it: {anonymous}"
);
assert!(
anonymous.contains("`shep restart log-rotate`"),
"every verdict ends in something the reader can run: {anonymous}"
);
let unattributed = stale_verdict("metrics", Silence::Unattributed);
for verdict in [&unreachable, &anonymous] {
assert!(
verdict.contains("`cargo install <crate> --force`"),
"an actionable verdict must carry the whole forced reinstall command: {verdict}"
);
}
assert!(
unattributed.contains("could not tell which process"),
"not knowing has to be said rather than papered over: {unattributed}"
);
assert!(
unattributed.contains("`shep bleats metrics`"),
"the one command that separates the two candidates: {unattributed}"
);
for verdict in [&unreachable, &anonymous, &unattributed] {
assert!(
!verdict.contains("the binary on disk cannot talk to this shep either"),
"the sentence that was asserted on every path is gone: {verdict}"
);
}
}
#[tokio::test(start_paused = true)]
async fn evidence_is_read_off_the_record_and_never_guessed() {
let contacts = PeerContacts::new();
tokio::time::advance(PEER_CONTACT_WARMUP * 2).await;
contacts.connected(11);
contacts.named_a_dog(12);
assert_eq!(
Silence::of(Some(10), &contacts),
Silence::Unreachable { pid: 10 }
);
assert_eq!(
Silence::of(Some(11), &contacts),
Silence::Anonymous { pid: 11 }
);
assert_eq!(
Silence::of(Some(12), &contacts),
Silence::Unattributed,
"a pid that named a dog and is silent anyway is a contradiction, not a diagnosis"
);
assert_eq!(
Silence::of(None, &contacts),
Silence::Unattributed,
"no pid is no attribution, which is a different answer from no contact"
);
}
#[tokio::test]
async fn shep_s_own_account_of_a_dog_reaches_that_dog_s_log() {
let h = crate::testing::harness(vec![ProcScript::never_exits()]);
start_test_dog(&h.ctx, "log-rotate").await;
let info = h
.ctx
.supervisor
.list()
.await
.into_iter()
.find(|info| info.name == "log-rotate")
.expect("the dog fixture must be listed");
let err_log = info
.err_file
.clone()
.expect("a dog's log paths are resolved");
let (out_tx, mut following) = tokio::sync::mpsc::channel(16);
let forwarder = crate::bus::spawn_forwarder(
&h.ctx.events,
crate::bus::TopicFilter::new(&["log.*".to_string()]).unwrap(),
out_tx,
);
narrate(&h.ctx.events, &info, "shep did a thing worth saying").await;
let written = std::fs::read_to_string(&err_log).expect("the narration must reach the log");
let line = written
.strip_suffix('\n')
.expect("one whole line, newline included");
assert!(
line.ends_with("[shep] shep did a thing worth saying"),
"the line must be marked as shep's voice, not the dog's: {line:?}"
);
let (stamp, rest) = line.split_at(shep_core::logstamp::LOG_STAMP_BYTES);
assert_eq!(
rest, "[shep] shep did a thing worth saying",
"the stamp is the same fixed-width prefix every other line carries: {line:?}"
);
chrono::DateTime::parse_from_rfc3339(stamp.trim_end())
.unwrap_or_else(|err| panic!("{stamp:?} must parse as RFC 3339: {err}"));
let frame = tokio::time::timeout(Duration::from_secs(5), following.recv())
.await
.expect("a follower must be told inside the budget")
.expect("the forwarder must deliver rather than end");
match shep_core::protocol::decode_frame::<BusEvent>(&frame).unwrap() {
BusEvent::LogErr { id, line } => {
assert_eq!(id, info.id, "the line belongs to the dog it is about");
assert_eq!(
line, "[shep] shep did a thing worth saying",
"a follower sees the marker and not the file's stamp"
);
}
other => panic!("narration must reach a follower as a log line, got {other:?}"),
}
forwarder.abort();
}
}