use core::fmt;
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex, PoisonError};
use serde::{Deserialize, Serialize};
use tempfile::NamedTempFile;
use tokio::sync::Notify;
use tokio::sync::broadcast::{self, error::RecvError};
use tokio::task::JoinHandle;
use tokio::time::{Duration, Instant, sleep_until};
use shep_core::config::graph::{BootPlan, render_cycle};
use shep_core::config::{AppConfig, NormalizeError, ResolvedApp, normalize};
use shep_core::protocol::{BusEvent, ProcessInfo};
use shep_core::status::ProcStatus;
use crate::bus::{Bus, SharedEvent};
use crate::supervisor::{BatchPolicy, SupervisorHandle};
pub(crate) const SNAPSHOT_VERSION: u32 = 1;
pub(crate) const SNAPSHOT_DEBOUNCE_MS: u64 = 250;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct FlockSnapshot {
pub version: u32,
pub saved_at_ms: u64,
pub apps: Vec<SavedApp>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct SavedApp {
pub app: AppConfig,
pub instances_running: u32,
}
#[derive(Debug, Clone, Default)]
pub(crate) struct FlockRegistry {
apps: Arc<Mutex<BTreeMap<String, AppConfig>>>,
dirty: Arc<Notify>,
}
impl FlockRegistry {
#[must_use]
pub(crate) fn new() -> Self {
Self::default()
}
pub(crate) fn record(&self, apps: &[ResolvedApp]) {
let mut map = self.apps.lock().unwrap_or_else(PoisonError::into_inner);
for app in apps {
map.insert(app.config().name.clone(), app.config().clone());
}
drop(map);
self.dirty.notify_one();
}
#[cfg(unix)]
pub(crate) fn record_config(&self, config: &AppConfig) {
self.apps
.lock()
.unwrap_or_else(PoisonError::into_inner)
.insert(config.name.clone(), config.clone());
self.dirty.notify_one();
}
pub(crate) fn clear(&self) {
self.apps
.lock()
.unwrap_or_else(PoisonError::into_inner)
.clear();
self.dirty.notify_one();
}
pub(crate) fn depends_on_by_name(&self) -> BTreeMap<String, Vec<String>> {
self.apps
.lock()
.unwrap_or_else(PoisonError::into_inner)
.iter()
.map(|(name, app)| (name.clone(), app.depends_on.clone()))
.collect()
}
pub(crate) fn timeouts_of(&self, name: &str) -> Option<(Duration, Duration)> {
self.apps
.lock()
.unwrap_or_else(PoisonError::into_inner)
.get(name)
.map(|app| {
(
app.listen_timeout.as_duration(),
app.graceful_timeout.as_duration(),
)
})
}
#[must_use]
pub(crate) fn roll(&self, infos: &[ProcessInfo], now_ms: u64) -> FlockSnapshot {
let mut apps = self.apps.lock().unwrap_or_else(PoisonError::into_inner);
apps.retain(|name, _| infos.iter().any(|info| &info.name == name));
let saved = apps
.iter()
.map(|(name, app)| SavedApp {
app: app.clone(),
instances_running: u32::try_from(
infos
.iter()
.filter(|i| &i.name == name && is_running(i.status))
.count(),
)
.unwrap_or(u32::MAX),
})
.collect();
FlockSnapshot {
version: SNAPSHOT_VERSION,
saved_at_ms: now_ms,
apps: saved,
}
}
}
fn is_running(status: ProcStatus) -> bool {
matches!(
status,
ProcStatus::Online | ProcStatus::Starting | ProcStatus::WaitingRestart
)
}
#[non_exhaustive]
#[derive(Debug)]
pub enum SnapshotError {
NoParent(PathBuf),
Encode(serde_json::Error),
Io(std::io::Error),
Parse(String),
}
impl fmt::Display for SnapshotError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::NoParent(path) => {
write!(f, "roll path `{}` has no parent directory", path.display())
}
Self::Encode(err) => write!(f, "muster roll failed to serialize: {err}"),
Self::Io(err) => write!(f, "muster roll I/O failed: {err}"),
Self::Parse(msg) => write!(f, "muster roll is unreadable: {msg}"),
}
}
}
impl core::error::Error for SnapshotError {
fn source(&self) -> Option<&(dyn core::error::Error + 'static)> {
match self {
Self::Encode(err) => Some(err),
Self::Io(err) => Some(err),
Self::NoParent(_) | Self::Parse(_) => None,
}
}
}
impl From<serde_json::Error> for SnapshotError {
fn from(source: serde_json::Error) -> Self {
Self::Encode(source)
}
}
impl From<std::io::Error> for SnapshotError {
fn from(source: std::io::Error) -> Self {
Self::Io(source)
}
}
pub(crate) fn write_atomic(path: &Path, snapshot: &FlockSnapshot) -> Result<(), SnapshotError> {
use std::io::Write;
let parent = path
.parent()
.ok_or_else(|| SnapshotError::NoParent(path.to_path_buf()))?;
let json = serde_json::to_vec_pretty(snapshot)?;
let mut tmp = NamedTempFile::new_in(parent)?;
tmp.write_all(&json)?;
tmp.as_file().sync_all()?;
tmp.persist(path)
.map_err(|err| SnapshotError::Io(err.error))?;
shep_core::atomic_file::sync_dir(parent)?;
Ok(())
}
pub fn read(path: &Path) -> Result<FlockSnapshot, SnapshotError> {
let bytes = std::fs::read(path)?;
let snapshot: FlockSnapshot =
serde_json::from_slice(&bytes).map_err(|err| SnapshotError::Parse(err.to_string()))?;
if snapshot.version != SNAPSHOT_VERSION {
return Err(SnapshotError::Parse(format!(
"roll schema version {} is not one this daemon knows (expected {SNAPSHOT_VERSION})",
snapshot.version
)));
}
Ok(snapshot)
}
#[derive(Debug)]
pub(crate) struct Restorable {
pub(crate) members: Vec<ResolvedApp>,
pub(crate) to_start: Vec<ResolvedApp>,
pub(crate) rejected: Vec<(String, NormalizeError)>,
}
#[must_use]
pub(crate) fn restorable(snapshot: FlockSnapshot) -> Restorable {
let mut members = Vec::new();
let mut to_start = Vec::new();
let mut rejected = Vec::new();
for saved in snapshot.apps {
let name = saved.app.name.clone();
let was_up = saved.instances_running > 0;
let autostart = saved.app.autostart;
match normalize(saved.app) {
Ok(resolved) => {
if was_up && autostart {
to_start.push(resolved.clone());
}
members.push(resolved);
}
Err(err) => rejected.push((name, err)),
}
}
Restorable {
members,
to_start,
rejected,
}
}
pub(crate) async fn muster(
path: &Path,
registry: &FlockRegistry,
supervisor: &SupervisorHandle,
events: &Bus,
dogs: &[String],
boot_first_dogs: &[String],
) -> Result<Vec<String>, SnapshotError> {
let saved = match read(path) {
Ok(saved) => saved,
Err(SnapshotError::Io(err)) if err.kind() == std::io::ErrorKind::NotFound => {
return Ok(Vec::new());
}
Err(err) => return Err(err),
};
let restorable = restorable(saved);
for (name, err) in &restorable.rejected {
tracing::warn!(name, %err, "muster roll entry rejected on restore");
}
if restorable.members.is_empty() {
return Ok(Vec::new());
}
let restored: Vec<String> = restorable
.members
.iter()
.map(|app| app.config().name.clone())
.collect();
let running = supervisor.list_checked().await.unwrap_or_default();
let known = |app: &ResolvedApp| running.iter().any(|info| info.name == app.config().name);
warn_about_dogs_holding_sheep_names(&restorable.members, &running);
let starting: Vec<&str> = restorable
.to_start
.iter()
.map(|app| app.config().name.as_str())
.collect();
let members: Vec<ResolvedApp> = restorable
.members
.iter()
.filter(|app| !known(app) && !starting.contains(&app.config().name.as_str()))
.cloned()
.collect();
if !members.is_empty() {
registry.record(&members);
if let Err(err) = supervisor.register_at_rest(members).await {
tracing::warn!(%err, "muster roll restore could not register one or more apps");
}
}
let to_start: Vec<ResolvedApp> = restorable
.to_start
.into_iter()
.filter(|app| !known(app))
.collect();
if to_start.is_empty() {
return Ok(restored);
}
registry.record(&to_start);
let plan = crate::boot_order::plan_for(&to_start, dogs, boot_first_dogs);
warn_about_the_graph(
&plan,
&to_start,
&restorable.members,
&running,
dogs,
boot_first_dogs,
);
if let Err(err) = crate::boot_order::start_in_stages(
&plan,
&to_start,
supervisor,
events,
BatchPolicy::PerApp,
)
.await
{
tracing::warn!(%err, "muster roll restore could not start one or more apps");
}
Ok(restored)
}
fn warn_about_dogs_holding_sheep_names(members: &[ResolvedApp], running: &[ProcessInfo]) {
for app in members {
let name = app.config().name.as_str();
let Some(source) = running
.iter()
.find(|info| info.name == name)
.and_then(|info| info.dog.as_ref())
else {
continue;
};
tracing::warn!(
name,
dog = ?source,
"a dog holds this name, so the roll's sheep of that name is not restored"
);
}
}
fn is_up(running: &[ProcessInfo], name: &str) -> bool {
running.iter().any(|info| {
info.name == name && !matches!(info.status, ProcStatus::Stopped | ProcStatus::Errored)
})
}
fn warn_about_the_graph(
plan: &BootPlan,
to_start: &[ResolvedApp],
members: &[ResolvedApp],
running: &[ProcessInfo],
dogs: &[String],
boot_first_dogs: &[String],
) {
let opted_out: BTreeSet<&str> = members
.iter()
.filter(|app| !app.config().autostart)
.map(|app| app.config().name.as_str())
.collect();
let restorable: BTreeSet<&str> = members
.iter()
.map(|app| app.config().name.as_str())
.collect();
for unresolved in &plan.unresolved {
if restorable.contains(unresolved.missing.as_str()) {
continue;
}
tracing::warn!(
sheep = %unresolved.dependent,
missing = %unresolved.missing,
"a dependency names nothing this flock has; starting without it"
);
}
for app in to_start {
for target in &app.config().depends_on {
if opted_out.contains(target.as_str()) {
tracing::warn!(
sheep = %app.config().name,
dependency = %target,
"the dependency sets autostart = false; starting without it"
);
}
}
}
let unpromoted: BTreeSet<&str> = dogs
.iter()
.map(String::as_str)
.filter(|dog| !boot_first_dogs.iter().any(|first| first == dog))
.filter(|dog| !is_up(running, dog))
.filter(|dog| !to_start.iter().any(|app| app.config().name == *dog))
.collect();
for app in to_start {
for target in &app.config().depends_on {
if unpromoted.contains(target.as_str()) {
tracing::warn!(
sheep = %app.config().name,
dog = %target,
"the dependency is a dog outside [daemon] boot_first_dogs, \
which starts after the whole flock; starting without it"
);
}
}
}
for cycle in &plan.cycles {
tracing::warn!(
cycle = %render_cycle(cycle),
"a dependency cycle; those sheep start last, in no particular order"
);
}
if let Some(stuck) = cyclic_stage(plan) {
let named: BTreeSet<&str> = plan.cycles.iter().flatten().map(String::as_str).collect();
if stuck.iter().any(|name| !named.contains(name.as_str())) {
let names: Vec<&str> = stuck.iter().map(String::as_str).collect();
tracing::warn!(
stuck = ?names,
"every sheep a dependency cycle holds; each of them starts last"
);
}
}
}
fn cyclic_stage(plan: &BootPlan) -> Option<&Vec<String>> {
let member = plan.cycles.first()?.first()?;
plan.stages.iter().find(|stage| stage.contains(member))
}
fn is_state_change(event: &BusEvent) -> bool {
matches!(event, BusEvent::Process { .. })
}
#[derive(Debug)]
pub(crate) struct SnapshotWriter {
handle: JoinHandle<()>,
#[allow(dead_code, reason = "read by this crate's own tests through `writes`")]
writes: Arc<AtomicU64>,
}
impl SnapshotWriter {
#[inline]
#[must_use]
#[allow(dead_code, reason = "called by this module's own tests")]
pub(crate) fn writes(&self) -> u64 {
self.writes.load(Ordering::SeqCst)
}
pub(crate) async fn stop(self) {
self.handle.abort();
let _ = self.handle.await;
}
}
pub(crate) fn spawn_snapshot_writer(
path: PathBuf,
supervisor: SupervisorHandle,
registry: FlockRegistry,
events: broadcast::Receiver<SharedEvent>,
) -> SnapshotWriter {
let writes = Arc::new(AtomicU64::new(0));
let task_writes = Arc::clone(&writes);
let dirty = Arc::clone(®istry.dirty);
let handle = tokio::spawn(run_writer(
path,
supervisor,
registry,
events,
dirty,
task_writes,
));
SnapshotWriter { handle, writes }
}
async fn run_writer(
path: PathBuf,
supervisor: SupervisorHandle,
registry: FlockRegistry,
mut events: broadcast::Receiver<SharedEvent>,
dirty: Arc<Notify>,
writes: Arc<AtomicU64>,
) {
let mut deadline: Option<Instant> = None;
loop {
tokio::select! {
received = events.recv() => match received {
Ok(event) => if is_state_change(&event) && deadline.is_none() {
deadline = Some(Instant::now() + Duration::from_millis(SNAPSHOT_DEBOUNCE_MS));
},
Err(RecvError::Lagged(_)) => if deadline.is_none() {
deadline = Some(Instant::now() + Duration::from_millis(SNAPSHOT_DEBOUNCE_MS));
},
Err(RecvError::Closed) => break,
},
() = dirty.notified() => if deadline.is_none() {
deadline = Some(Instant::now() + Duration::from_millis(SNAPSHOT_DEBOUNCE_MS));
},
() = sleep_until(deadline.unwrap_or_else(Instant::now)), if deadline.is_some() => {
deadline = None;
write_now(&path, &supervisor, ®istry, &writes).await;
}
}
}
}
async fn write_now(
path: &Path,
supervisor: &SupervisorHandle,
registry: &FlockRegistry,
writes: &AtomicU64,
) {
let Ok(infos) = supervisor.list_checked().await else {
return;
};
let roll = registry.roll(&infos, crate::now_ms()); match write_atomic(path, &roll) {
Ok(()) => {
writes.fetch_add(1, Ordering::SeqCst);
}
Err(err) => tracing::warn!(%err, "muster roll write failed"),
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::fake::{ProcScript, ScriptedRunner};
use crate::supervisor::spawn_supervisor;
use crate::testing::{capture_logs, test_paths};
use shep_core::config::graph::{BootNode, NodeKind, plan};
use shep_core::config::{AppConfig, normalize};
use shep_core::protocol::{BusEvent, DogSource, ProcessEventKind, ProcessInfo};
use shep_core::status::ProcStatus;
use shep_core::values::UpDuration;
use std::time::Duration;
fn info(id: u32, name: &str, status: ProcStatus) -> ProcessInfo {
ProcessInfo::builder(id, name, status)
.pid(Some(1000 + id))
.out_file(Some(format!("/logs/{name}-0-out.log")))
.err_file(Some(format!("/logs/{name}-0-err.log")))
.build()
}
#[test]
fn a_resolved_secret_never_reaches_the_muster_roll() {
const SENTINEL: &str = "hunter2-that-must-never-reach-disk";
let mut config = AppConfig::minimal("web", "./srv");
config
.env
.insert("PW".to_string(), "{{secret:DB_PASSWORD}}".to_string());
let app = normalize(config).unwrap();
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let view = shep_core::secrets::SecretView::new(
"production".to_string(),
std::collections::BTreeMap::from([(
"DB_PASSWORD".to_string(),
std::collections::BTreeMap::from([(
"production".to_string(),
SENTINEL.to_string(),
)]),
)]),
shep_core::secrets::ProviderCache::default(),
);
let spec = crate::assemble::assemble(&app, 0, &paths, None, &view).unwrap();
assert_eq!(
spec.env["PW"], SENTINEL,
"the fixture must really resolve, or this case proves nothing"
);
let registry = FlockRegistry::new();
registry.record(&[app]);
let roll = registry.roll(&[info(0, "web", ProcStatus::Online)], 0);
let json = serde_json::to_string(&roll).unwrap();
assert!(!json.contains(SENTINEL), "the roll carries a value: {json}");
assert!(
json.contains("{{secret:DB_PASSWORD}}"),
"the reference is what it carries instead: {json}"
);
let path = dir.path().join("flock.json");
write_atomic(&path, &roll).unwrap();
let on_disk = std::fs::read_to_string(&path).unwrap();
assert!(
!on_disk.contains(SENTINEL),
"the roll on disk carries a value: {on_disk}"
);
}
#[test]
fn roll_counts_running_instances_and_prunes_deleted_names() {
let registry = FlockRegistry::new();
let web = normalize(AppConfig::minimal("web", "./srv")).unwrap();
let job = normalize(AppConfig::minimal("job", "./job")).unwrap();
registry.record(&[web, job]);
let infos = [
info(0, "web", ProcStatus::Online),
info(1, "web", ProcStatus::WaitingRestart),
info(2, "web", ProcStatus::Stopped),
info(3, "web", ProcStatus::Stopping),
]; let roll = registry.roll(&infos, 1_700_000_000_000);
assert_eq!(roll.version, SNAPSHOT_VERSION);
assert_eq!(roll.saved_at_ms, 1_700_000_000_000);
assert_eq!(roll.apps.len(), 1, "a name with no live entry is pruned");
assert_eq!(roll.apps[0].app.name, "web");
assert_eq!(roll.apps[0].instances_running, 2);
assert_eq!(registry.roll(&infos, 0).apps.len(), 1);
}
#[test]
fn write_atomic_round_trips_with_no_leftovers() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("flock.json");
let registry = FlockRegistry::new();
registry.record(&[normalize(AppConfig::minimal("web", "./srv")).unwrap()]);
let roll = registry.roll(&[info(0, "web", ProcStatus::Online)], 42);
write_atomic(&path, &roll).unwrap();
write_atomic(&path, &roll).unwrap();
assert_eq!(read(&path).unwrap(), roll);
let entries: Vec<_> = std::fs::read_dir(dir.path()).unwrap().collect();
assert_eq!(
entries.len(),
1,
"no temp file may survive a completed write"
);
}
#[cfg(unix)]
#[test]
fn write_atomic_is_owner_only_on_unix() {
use std::os::unix::fs::PermissionsExt;
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("flock.json");
let registry = FlockRegistry::new();
registry.record(&[normalize(AppConfig::minimal("web", "./srv")).unwrap()]);
let roll = registry.roll(&[info(0, "web", ProcStatus::Online)], 42);
write_atomic(&path, &roll).unwrap();
let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777;
assert_eq!(mode, 0o600, "the muster roll holds app env in cleartext");
}
#[test]
fn read_rejects_corrupt_json_and_unknown_schema_versions() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("flock.json");
std::fs::write(&path, b"{not json").unwrap();
assert!(matches!(read(&path), Err(SnapshotError::Parse { .. })));
let future = format!(
"{{\"version\":{},\"saved_at_ms\":0,\"apps\":[]}}",
SNAPSHOT_VERSION + 1
);
std::fs::write(&path, future.as_bytes()).unwrap();
assert!(matches!(read(&path), Err(SnapshotError::Parse { .. })));
}
#[test]
fn restorable_keeps_every_member_and_starts_only_what_was_up() {
let mut stopped = AppConfig::minimal("stopped", "./s");
stopped.instances = 1;
let mut opted_out = AppConfig::minimal("manual", "./m");
opted_out.autostart = false;
let roll = FlockSnapshot {
version: SNAPSHOT_VERSION,
saved_at_ms: 0,
apps: vec![
SavedApp {
app: AppConfig::minimal("web", "./srv"),
instances_running: 2,
},
SavedApp {
app: stopped,
instances_running: 0,
},
SavedApp {
app: opted_out,
instances_running: 1,
},
],
};
let restorable = restorable(roll);
let members: Vec<&str> = restorable
.members
.iter()
.map(|a| a.config().name.as_str())
.collect();
assert_eq!(
members,
["web", "stopped", "manual"],
"every entry that normalizes belongs to the flock, running or not"
);
let starting: Vec<&str> = restorable
.to_start
.iter()
.map(|a| a.config().name.as_str())
.collect();
assert_eq!(
starting,
["web"],
"only the sheep that was up and opts into autostart is started"
);
assert!(restorable.rejected.is_empty());
}
#[test]
fn a_sheep_saved_while_stopped_is_still_a_member() {
let roll = FlockSnapshot {
version: SNAPSHOT_VERSION,
saved_at_ms: 0,
apps: vec![SavedApp {
app: AppConfig::minimal("zeus-auth", "./zeus-auth"),
instances_running: 0,
}],
};
let restorable = restorable(roll);
assert_eq!(restorable.members.len(), 1, "stopping is not forgetting");
assert_eq!(restorable.members[0].config().name, "zeus-auth");
assert!(
restorable.to_start.is_empty(),
"but it stays stopped: it was not running when the roll was written"
);
}
#[test]
fn restorable_reports_a_hand_edited_invalid_app_instead_of_aborting() {
let mut broken = AppConfig::minimal("broken", "./b");
broken.instances = 0; let roll = FlockSnapshot {
version: SNAPSHOT_VERSION,
saved_at_ms: 0,
apps: vec![
SavedApp {
app: broken,
instances_running: 1,
},
SavedApp {
app: AppConfig::minimal("web", "./srv"),
instances_running: 1,
},
],
};
let restorable = restorable(roll);
assert_eq!(
restorable.members.len(),
1,
"one bad entry must not sink the muster"
);
assert_eq!(
restorable.rejected,
vec![(
"broken".to_string(),
shep_core::config::NormalizeError::ZeroInstances
)]
);
}
#[test]
fn debug_does_not_leak_env_values() {
let mut app = AppConfig::minimal("web", "./srv");
app.env
.insert("DATABASE_URL".to_string(), "postgres://secret".to_string());
let roll = FlockSnapshot {
version: SNAPSHOT_VERSION,
saved_at_ms: 0,
apps: vec![SavedApp {
app,
instances_running: 1,
}],
};
let rendered = format!("{roll:?}");
assert!(!rendered.contains("postgres://secret"), "{rendered}");
assert!(rendered.contains("<1 vars>"), "{rendered}");
}
#[tokio::test(start_paused = true)]
async fn muster_restores_both_and_starts_only_the_one_that_was_up() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
std::fs::create_dir_all(paths.snapshot.parent().unwrap()).unwrap();
let roll = FlockSnapshot {
version: SNAPSHOT_VERSION,
saved_at_ms: 0,
apps: vec![
SavedApp {
app: AppConfig::minimal("up", "./srv"),
instances_running: 1,
},
SavedApp {
app: AppConfig::minimal("down", "./srv"),
instances_running: 0,
},
],
};
write_atomic(&paths.snapshot, &roll).unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let handle = spawn_supervisor(
ScriptedRunner::new(vec![ProcScript::never_exits()]),
paths.clone(),
events.clone(),
);
let registry = FlockRegistry::new();
let restored = muster(&paths.snapshot, ®istry, &handle, &events, &[], &[])
.await
.unwrap();
assert_eq!(
restored,
vec!["up".to_string(), "down".to_string()],
"both are restored to the flock; only one of them runs"
);
let mut listed = handle.list().await;
listed.sort_by(|a, b| a.name.cmp(&b.name));
let seen: Vec<(&str, ProcStatus)> =
listed.iter().map(|i| (i.name.as_str(), i.status)).collect();
assert_eq!(
seen,
vec![("down", ProcStatus::Stopped), ("up", ProcStatus::Online)],
"the sheep that was down is listed and stopped, not missing"
);
handle.shutdown().await;
}
#[tokio::test(start_paused = true)]
async fn a_bad_saved_app_does_not_take_the_apps_after_it_down() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
std::fs::create_dir_all(paths.snapshot.parent().unwrap()).unwrap();
let saved = |name: &str| SavedApp {
app: AppConfig::minimal(name, "./srv"),
instances_running: 1,
};
let roll = FlockSnapshot {
version: SNAPSHOT_VERSION,
saved_at_ms: 0,
apps: vec![saved("a-good"), saved("b-bad"), saved("c-good")],
};
write_atomic(&paths.snapshot, &roll).unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let handle = spawn_supervisor(
ScriptedRunner::new(vec![ProcScript::never_exits(); 2]).failing_to_spawn(&["b-bad"]),
paths.clone(),
events.clone(),
);
let registry = FlockRegistry::new();
let restored = muster(&paths.snapshot, ®istry, &handle, &events, &[], &[])
.await
.unwrap();
assert_eq!(
restored,
vec![
"a-good".to_string(),
"b-bad".to_string(),
"c-good".to_string()
]
);
let mut listed = handle.list().await;
listed.sort_by(|a, b| a.name.cmp(&b.name));
let seen: Vec<(&str, ProcStatus)> =
listed.iter().map(|i| (i.name.as_str(), i.status)).collect();
assert_eq!(
seen,
vec![
("a-good", ProcStatus::Online),
("b-bad", ProcStatus::Errored),
("c-good", ProcStatus::Online),
],
"every app after the broken one must still get its turn, and the \
broken one must be visible rather than absent"
);
handle.shutdown().await;
}
#[tokio::test(start_paused = true)]
async fn one_unstartable_saved_app_does_not_keep_the_rest_of_the_flock_down() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
std::fs::create_dir_all(paths.snapshot.parent().unwrap()).unwrap();
let roll = FlockSnapshot {
version: SNAPSHOT_VERSION,
saved_at_ms: 0,
apps: vec![
SavedApp {
app: AppConfig::minimal("good", "./srv"),
instances_running: 1,
},
SavedApp {
app: AppConfig::minimal("gone", "./deleted-by-a-rebuild"),
instances_running: 1,
},
],
};
write_atomic(&paths.snapshot, &roll).unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let handle = spawn_supervisor(
ScriptedRunner::new(vec![ProcScript::never_exits()])
.refusing(&["gone"])
.failing_to_spawn(&["gone"]),
paths.clone(),
events.clone(),
);
let registry = FlockRegistry::new();
let restored = muster(&paths.snapshot, ®istry, &handle, &events, &[], &[])
.await
.unwrap();
assert_eq!(restored, vec!["good".to_string(), "gone".to_string()]);
let mut listed = handle.list().await;
listed.sort_by(|a, b| a.name.cmp(&b.name));
let seen: Vec<(&str, ProcStatus)> =
listed.iter().map(|i| (i.name.as_str(), i.status)).collect();
assert_eq!(
seen,
vec![("gone", ProcStatus::Errored), ("good", ProcStatus::Online)],
"the app that could still run must come up, and the one that \
could not must be visible rather than absent"
);
handle.shutdown().await;
}
#[tokio::test(start_paused = true)]
async fn muster_leaves_an_app_the_flock_already_has_where_it_stands() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
std::fs::create_dir_all(paths.snapshot.parent().unwrap()).unwrap();
let roll = FlockSnapshot {
version: SNAPSHOT_VERSION,
saved_at_ms: 0,
apps: vec![SavedApp {
app: AppConfig::minimal("web", "./srv"),
instances_running: 1,
}],
};
write_atomic(&paths.snapshot, &roll).unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let handle = spawn_supervisor(
ScriptedRunner::new(vec![ProcScript::never_exits()]),
paths.clone(),
events.clone(),
);
let registry = FlockRegistry::new();
let app = normalize(AppConfig::minimal("web", "./srv")).unwrap();
registry.record(std::slice::from_ref(&app));
handle.start(vec![app]).await.unwrap();
let restored = muster(&paths.snapshot, ®istry, &handle, &events, &[], &[])
.await
.unwrap();
assert_eq!(
restored,
vec!["web".to_string()],
"the roll's app is restored whether or not this call is what started it"
);
let listed = handle.list().await;
assert_eq!(listed.len(), 1, "one instance of a one-instance app");
assert_eq!(listed[0].status, ProcStatus::Online);
handle.shutdown().await;
}
fn roll_of(apps: Vec<AppConfig>) -> FlockSnapshot {
FlockSnapshot {
version: SNAPSHOT_VERSION,
saved_at_ms: 0,
apps: apps
.into_iter()
.map(|app| SavedApp {
app,
instances_running: 1,
})
.collect(),
}
}
fn start_order(rx: &mut broadcast::Receiver<SharedEvent>) -> Vec<String> {
let mut order = Vec::new();
while let Ok(event) = rx.try_recv() {
if let BusEvent::Process {
event: ProcessEventKind::Start,
info,
..
} = &*event
{
order.push(info.name.clone());
}
}
order
}
#[tokio::test(start_paused = true)]
async fn a_restore_starts_the_roll_in_dependency_order() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
std::fs::create_dir_all(paths.snapshot.parent().unwrap()).unwrap();
let mut db = AppConfig::minimal("db", "./srv");
db.listen_timeout = UpDuration::from_millis(50);
let mut api = AppConfig::minimal("api", "./srv");
api.depends_on = vec!["db".to_string()];
write_atomic(&paths.snapshot, &roll_of(vec![api, db])).unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let handle = spawn_supervisor(
ScriptedRunner::new(vec![ProcScript::never_exits(); 2]),
paths.clone(),
events.clone(),
);
let registry = FlockRegistry::new();
let mut seen = events.subscribe();
muster(&paths.snapshot, ®istry, &handle, &events, &[], &[])
.await
.unwrap();
assert_eq!(
start_order(&mut seen),
vec!["db", "api"],
"the roll's order must not decide boot order"
);
handle.shutdown().await;
}
#[tokio::test(start_paused = true)]
async fn a_cyclic_roll_still_brings_the_flock_up() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
std::fs::create_dir_all(paths.snapshot.parent().unwrap()).unwrap();
let mut a = AppConfig::minimal("a", "./srv");
a.depends_on = vec!["b".to_string()];
let mut b = AppConfig::minimal("b", "./srv");
b.depends_on = vec!["a".to_string()];
write_atomic(&paths.snapshot, &roll_of(vec![a, b])).unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let handle = spawn_supervisor(
ScriptedRunner::new(vec![ProcScript::never_exits(); 2]),
paths.clone(),
events.clone(),
);
let registry = FlockRegistry::new();
let restored = muster(&paths.snapshot, ®istry, &handle, &events, &[], &[])
.await
.expect("a cycle must not refuse the restore");
assert_eq!(restored, vec!["a".to_string(), "b".to_string()]);
let mut listed = handle.list().await;
listed.sort_by(|left, right| left.name.cmp(&right.name));
let seen: Vec<(&str, ProcStatus)> = listed
.iter()
.map(|info| (info.name.as_str(), info.status))
.collect();
assert_eq!(
seen,
vec![("a", ProcStatus::Online), ("b", ProcStatus::Online)],
"both members of the knot run; neither waits for the other"
);
handle.shutdown().await;
}
#[tokio::test(start_paused = true)]
async fn a_dependency_with_autostart_off_is_warned_about_and_skipped() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
std::fs::create_dir_all(paths.snapshot.parent().unwrap()).unwrap();
let mut db = AppConfig::minimal("db", "./srv");
db.autostart = false;
let mut api = AppConfig::minimal("api", "./srv");
api.depends_on = vec!["db".to_string()];
write_atomic(&paths.snapshot, &roll_of(vec![db, api])).unwrap();
let (events, _rx) = crate::bus::test_bus(64);
let handle = spawn_supervisor(
ScriptedRunner::new(vec![ProcScript::never_exits()]),
paths.clone(),
events.clone(),
);
let registry = FlockRegistry::new();
let mut seen = events.subscribe();
muster(&paths.snapshot, ®istry, &handle, &events, &[], &[])
.await
.unwrap();
assert_eq!(
start_order(&mut seen),
vec!["api"],
"db opted out; api starts without it"
);
handle.shutdown().await;
}
fn dependant(name: &str, deps: &[&str]) -> ResolvedApp {
let mut app = AppConfig::minimal(name, "./srv");
app.depends_on = deps.iter().map(|dep| (*dep).to_string()).collect();
normalize(app).expect("a minimal app with edges normalizes")
}
#[test]
fn a_dependency_the_roll_still_holds_is_not_called_missing() {
let api = dependant("api", &["db"]);
let db = normalize(AppConfig::minimal("db", "./srv")).expect("a minimal app normalizes");
let to_start = vec![api.clone()];
let plan = crate::boot_order::plan_for(&to_start, &[], &[]);
assert_eq!(
plan.unresolved.len(),
1,
"the plan only ever sees what this restore starts"
);
let held = capture_logs(|| {
warn_about_the_graph(&plan, &to_start, &[db, api.clone()], &[], &[], &[])
});
assert!(!held.contains("names nothing this flock has"), "{held}");
let absent = capture_logs(|| warn_about_the_graph(&plan, &to_start, &[api], &[], &[], &[]));
assert!(absent.contains("names nothing this flock has"), "{absent}");
}
#[test]
fn a_dependency_on_a_dog_the_boot_never_promotes_is_warned_about() {
let to_start = vec![dependant("api", &["metrics"])];
let dogs = ["metrics".to_string()];
let late = capture_logs(|| {
let plan = crate::boot_order::plan_for(&to_start, &dogs, &[]);
warn_about_the_graph(&plan, &to_start, &to_start, &[], &dogs, &[]);
});
assert!(late.contains("starts after the whole flock"), "{late}");
let promoted = capture_logs(|| {
let plan = crate::boot_order::plan_for(&to_start, &dogs, &dogs);
warn_about_the_graph(&plan, &to_start, &to_start, &[], &dogs, &dogs);
});
assert!(
!promoted.contains("starts after the whole flock"),
"{promoted}"
);
}
#[test]
fn a_dependency_on_an_unpromoted_dog_that_is_already_running_is_not_warned_about() {
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
let dogs = ["metrics".to_string()];
let restore = |dog_running: bool| {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
std::fs::create_dir_all(paths.snapshot.parent().unwrap()).unwrap();
let mut api = AppConfig::minimal("api", "./srv");
api.depends_on = vec!["metrics".to_string()];
let roll = FlockSnapshot {
version: SNAPSHOT_VERSION,
saved_at_ms: 0,
apps: vec![SavedApp {
app: api,
instances_running: 1,
}],
};
write_atomic(&paths.snapshot, &roll).unwrap();
capture_logs(|| {
rt.block_on(async {
let (events, _rx) = crate::bus::test_bus(64);
let handle = spawn_supervisor(
ScriptedRunner::new(vec![
ProcScript::never_exits(),
ProcScript::never_exits(),
]),
paths.clone(),
events.clone(),
);
if dog_running {
let dog = normalize(AppConfig::minimal("metrics", "./srv"))
.expect("a minimal dog normalizes");
handle
.start_dog(dog, DogSource::BuiltIn)
.await
.expect("a scripted dog that never exits starts");
}
muster(
&paths.snapshot,
&FlockRegistry::new(),
&handle,
&events,
&dogs,
&[],
)
.await
.expect("the roll parses");
});
})
};
let live = restore(true);
assert!(!live.contains("starts after the whole flock"), "{live}");
let at_boot = restore(false);
assert!(
at_boot.contains("starts after the whole flock"),
"{at_boot}"
);
}
#[test]
fn a_dog_holding_a_saved_sheeps_name_is_warned_about() {
let members = vec![normalize(AppConfig::minimal("metrics", "./srv")).expect("normalizes")];
let as_dog = [ProcessInfo::builder(0, "metrics", ProcStatus::Online)
.dog(Some(DogSource::BuiltIn))
.build()];
let as_sheep = [ProcessInfo::builder(0, "metrics", ProcStatus::Online).build()];
let collided = capture_logs(|| warn_about_dogs_holding_sheep_names(&members, &as_dog));
assert!(collided.contains("is not restored"), "{collided}");
let plain = capture_logs(|| warn_about_dogs_holding_sheep_names(&members, &as_sheep));
assert!(!plain.contains("is not restored"), "{plain}");
}
#[test]
fn a_two_node_knot_is_not_reported_twice_over() {
let pair = vec![dependant("a", &["b"]), dependant("b", &["a"])];
let plan = crate::boot_order::plan_for(&pair, &[], &[]);
let two = capture_logs(|| warn_about_the_graph(&plan, &pair, &pair, &[], &[], &[]));
assert!(
!two.contains("every sheep a dependency cycle holds"),
"{two}"
);
let trio = vec![
dependant("a", &["b"]),
dependant("b", &["a", "c"]),
dependant("c", &["b"]),
];
let plan = crate::boot_order::plan_for(&trio, &[], &[]);
let three = capture_logs(|| warn_about_the_graph(&plan, &trio, &trio, &[], &[], &[]));
assert!(
three.contains("every sheep a dependency cycle holds"),
"{three}"
);
}
#[test]
fn the_cyclic_stage_names_every_sheep_in_the_knot() {
let node = |name: &str, deps: &[&str]| BootNode {
name: name.to_string(),
depends_on: deps.iter().map(|dep| (*dep).to_string()).collect(),
kind: NodeKind::Sheep,
};
let plan = plan(&[node("a", &["b"]), node("b", &["a", "c"]), node("c", &["b"])]);
assert_eq!(plan.cycles.len(), 1, "one knot, one representative path");
assert!(
plan.cycles[0].len() < 3,
"the point of the extra report is a member the path leaves out"
);
assert_eq!(
cyclic_stage(&plan),
Some(&vec!["a".to_string(), "b".to_string(), "c".to_string()]),
);
}
#[tokio::test(start_paused = true)]
async fn a_missing_roll_restores_nothing_without_failing() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
std::fs::create_dir_all(paths.snapshot.parent().unwrap()).unwrap();
assert!(!paths.snapshot.exists());
let (events, _rx) = crate::bus::test_bus(64);
let handle = spawn_supervisor(
ScriptedRunner::new(vec![ProcScript::never_exits()]),
paths.clone(),
events.clone(),
);
let registry = FlockRegistry::new();
assert_eq!(
muster(&paths.snapshot, ®istry, &handle, &events, &[], &[])
.await
.unwrap(),
Vec::<String>::new()
);
assert!(handle.list().await.is_empty());
handle.shutdown().await;
}
#[tokio::test(start_paused = true)]
async fn writer_coalesces_a_burst_into_one_write() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
std::fs::create_dir_all(&paths.home).unwrap();
let (events, _keep) = crate::bus::test_bus(64);
let supervisor = spawn_supervisor(
ScriptedRunner::new(vec![ProcScript::never_exits()]),
paths.clone(),
events.clone(),
);
let registry = FlockRegistry::new();
let app = normalize(AppConfig::minimal("web", "./srv")).unwrap();
registry.record(std::slice::from_ref(&app));
supervisor.start(vec![app]).await.unwrap();
let writer = spawn_snapshot_writer(
paths.snapshot.clone(),
supervisor.clone(),
registry,
events.subscribe(),
);
for event in [
ProcessEventKind::Exit,
ProcessEventKind::Restart,
ProcessEventKind::Online,
] {
events
.send(
BusEvent::Process {
event,
info: info(0, "web", ProcStatus::Online),
manually: false,
at_ms: 0,
}
.into(),
)
.unwrap();
}
tokio::time::sleep(Duration::from_millis(SNAPSHOT_DEBOUNCE_MS + 1)).await;
assert_eq!(writer.writes(), 1, "one debounce window is one write");
let roll = read(&paths.snapshot).unwrap();
assert_eq!(roll.apps.len(), 1);
assert_eq!(roll.apps[0].instances_running, 1);
writer.stop().await;
}
#[tokio::test(start_paused = true)]
async fn writer_schedules_a_write_for_a_recording_the_bus_says_nothing_about() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
std::fs::create_dir_all(&paths.home).unwrap();
let (events, _keep) = crate::bus::test_bus(64);
let supervisor = spawn_supervisor(
ScriptedRunner::new(vec![ProcScript::never_exits()]),
paths.clone(),
events.clone(),
);
let registry = FlockRegistry::new();
let app = normalize(AppConfig::minimal("web", "./srv")).unwrap();
supervisor.start(vec![app.clone()]).await.unwrap();
let writer = spawn_snapshot_writer(
paths.snapshot.clone(),
supervisor.clone(),
registry.clone(),
events.subscribe(),
);
assert!(!paths.snapshot.exists(), "nothing has recorded yet");
registry.record(std::slice::from_ref(&app));
tokio::time::sleep(Duration::from_millis(SNAPSHOT_DEBOUNCE_MS + 1)).await;
assert_eq!(writer.writes(), 1, "a recording leaves the roll dirty");
let roll = read(&paths.snapshot).unwrap();
assert_eq!(roll.apps.len(), 1);
writer.stop().await;
}
#[tokio::test(start_paused = true)]
async fn writer_ignores_log_traffic() {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
std::fs::create_dir_all(&paths.home).unwrap();
let (events, _keep) = crate::bus::test_bus(64);
let supervisor = spawn_supervisor(
ScriptedRunner::new(vec![ProcScript::never_exits()]),
paths.clone(),
events.clone(),
);
let writer = spawn_snapshot_writer(
paths.snapshot.clone(),
supervisor,
FlockRegistry::new(),
events.subscribe(),
);
for id in 0..50 {
events
.send(
BusEvent::LogOut {
id,
line: "chatty".to_string(),
}
.into(),
)
.unwrap();
}
tokio::time::sleep(Duration::from_millis(SNAPSHOT_DEBOUNCE_MS * 4)).await;
assert_eq!(writer.writes(), 0, "log lines must never rewrite the roll");
assert!(!paths.snapshot.exists());
writer.stop().await;
}
}