use core::fmt;
use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex, PoisonError};
use serde::{Deserialize, Serialize};
use tempfile::NamedTempFile;
use tokio::sync::Notify;
use tokio::sync::broadcast::{self, error::RecvError};
use tokio::task::JoinHandle;
use tokio::time::{Duration, Instant, sleep_until};
use shep_core::config::graph::{BootPlan, render_cycle};
use shep_core::config::{AppConfig, NormalizeError, ResolvedApp, normalize};
use shep_core::protocol::{BusEvent, ProcessInfo};
use shep_core::status::ProcStatus;
use crate::bus::{Bus, SharedEvent};
use crate::supervisor::{BatchPolicy, SupervisorHandle};
pub(crate) const SNAPSHOT_VERSION: u32 = 1;
pub(crate) const SNAPSHOT_DEBOUNCE_MS: u64 = 250;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct FlockSnapshot {
pub version: u32,
pub saved_at_ms: u64,
pub apps: Vec<SavedApp>,
}
impl FlockSnapshot {
#[must_use]
pub fn with_apps(apps: Vec<SavedApp>) -> Self {
Self {
version: SNAPSHOT_VERSION,
saved_at_ms: 0,
apps,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct SavedApp {
pub app: AppConfig,
pub instances_running: u32,
#[serde(default)]
pub disabled: bool,
}
#[derive(Debug, Clone, Default)]
pub(crate) struct FlockRegistry {
apps: Arc<Mutex<BTreeMap<String, AppConfig>>>,
dirty: Arc<Notify>,
}
impl FlockRegistry {
#[must_use]
pub(crate) fn new() -> Self {
Self::default()
}
pub(crate) fn record(&self, apps: &[ResolvedApp]) {
let mut map = self.apps.lock().unwrap_or_else(PoisonError::into_inner);
for app in apps {
map.insert(app.config().name.clone(), app.config().clone());
}
drop(map);
self.dirty.notify_one();
}
#[cfg(unix)]
pub(crate) fn record_config(&self, config: &AppConfig) {
self.apps
.lock()
.unwrap_or_else(PoisonError::into_inner)
.insert(config.name.clone(), config.clone());
self.dirty.notify_one();
}
pub(crate) fn touch(&self) {
self.dirty.notify_one();
}
pub(crate) fn clear(&self) {
self.apps
.lock()
.unwrap_or_else(PoisonError::into_inner)
.clear();
self.dirty.notify_one();
}
pub(crate) fn depends_on_by_name(&self) -> BTreeMap<String, Vec<String>> {
self.apps
.lock()
.unwrap_or_else(PoisonError::into_inner)
.iter()
.map(|(name, app)| (name.clone(), app.depends_on.clone()))
.collect()
}
pub(crate) fn timeouts_of(&self, name: &str) -> Option<(Duration, Duration)> {
self.apps
.lock()
.unwrap_or_else(PoisonError::into_inner)
.get(name)
.map(|app| {
(
app.listen_timeout.as_duration(),
app.graceful_timeout.as_duration(),
)
})
}
#[must_use]
pub(crate) fn roll(&self, infos: &[ProcessInfo], now_ms: u64) -> FlockSnapshot {
let mut apps = self.apps.lock().unwrap_or_else(PoisonError::into_inner);
let mut running: BTreeMap<&str, u32> = BTreeMap::new();
let mut parked: BTreeSet<&str> = BTreeSet::new();
for info in infos {
let up = running.entry(info.name.as_str()).or_default();
*up = up.saturating_add(u32::from(is_running(info.status)));
if info.disabled == Some(true) {
parked.insert(info.name.as_str());
}
}
apps.retain(|name, _| running.contains_key(name.as_str()));
let saved = apps
.iter()
.map(|(name, app)| SavedApp {
app: app.clone(),
instances_running: running.get(name.as_str()).copied().unwrap_or(0),
disabled: parked.contains(name.as_str()),
})
.collect();
FlockSnapshot {
saved_at_ms: now_ms,
..FlockSnapshot::with_apps(saved)
}
}
}
fn is_running(status: ProcStatus) -> bool {
matches!(
status,
ProcStatus::Online | ProcStatus::Starting | ProcStatus::WaitingRestart
)
}
#[non_exhaustive]
#[derive(Debug)]
pub enum SnapshotError {
NoParent(PathBuf),
Encode(serde_json::Error),
Io(std::io::Error),
Decode(serde_json::Error),
Parse(String),
}
impl fmt::Display for SnapshotError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::NoParent(path) => {
write!(f, "roll path `{}` has no parent directory", path.display())
}
Self::Encode(err) => write!(f, "muster roll failed to serialize: {err}"),
Self::Io(err) => write!(f, "muster roll I/O failed: {err}"),
Self::Decode(err) => write!(f, "muster roll is unreadable: {err}"),
Self::Parse(msg) => write!(f, "muster roll is unreadable: {msg}"),
}
}
}
impl core::error::Error for SnapshotError {
fn source(&self) -> Option<&(dyn core::error::Error + 'static)> {
match self {
Self::Encode(err) | Self::Decode(err) => Some(err),
Self::Io(err) => Some(err),
Self::NoParent(_) | Self::Parse(_) => None,
}
}
}
impl From<serde_json::Error> for SnapshotError {
fn from(source: serde_json::Error) -> Self {
Self::Encode(source)
}
}
impl From<std::io::Error> for SnapshotError {
fn from(source: std::io::Error) -> Self {
Self::Io(source)
}
}
pub(crate) fn write_atomic(path: &Path, snapshot: &FlockSnapshot) -> Result<(), SnapshotError> {
use std::io::Write;
let parent = path
.parent()
.ok_or_else(|| SnapshotError::NoParent(path.to_path_buf()))?;
let json = serde_json::to_vec_pretty(snapshot)?;
let mut tmp = NamedTempFile::new_in(parent)?;
tmp.write_all(&json)?;
shep_core::atomic_file::publish(tmp, path).map_err(SnapshotError::Io)
}
pub fn read(path: &Path) -> Result<FlockSnapshot, SnapshotError> {
let bytes = std::fs::read(path)?;
let snapshot: FlockSnapshot = serde_json::from_slice(&bytes).map_err(SnapshotError::Decode)?;
if snapshot.version != SNAPSHOT_VERSION {
return Err(SnapshotError::Parse(format!(
"roll schema version {} is not one this daemon knows (expected {SNAPSHOT_VERSION})",
snapshot.version
)));
}
Ok(snapshot)
}
#[derive(Debug)]
pub(crate) struct Restorable {
pub(crate) members: Vec<ResolvedApp>,
pub(crate) to_start: Vec<ResolvedApp>,
pub(crate) disabled: BTreeSet<String>,
pub(crate) rejected: Vec<(String, NormalizeError)>,
}
#[must_use]
pub(crate) fn restorable(snapshot: FlockSnapshot) -> Restorable {
let mut members = Vec::new();
let mut to_start = Vec::new();
let mut disabled = BTreeSet::new();
let mut rejected = Vec::new();
for saved in snapshot.apps {
let name = saved.app.name.clone();
let was_up = saved.instances_running > 0;
let autostart = saved.app.autostart;
match normalize(saved.app) {
Ok(resolved) => {
if saved.disabled {
disabled.insert(name);
} else if was_up && autostart {
to_start.push(resolved.clone());
}
members.push(resolved);
}
Err(err) => rejected.push((name, err)),
}
}
Restorable {
members,
to_start,
disabled,
rejected,
}
}
pub(crate) async fn muster(
path: &Path,
registry: &FlockRegistry,
supervisor: &SupervisorHandle,
events: &Bus,
dogs: &[String],
boot_first_dogs: &[String],
) -> Result<Vec<String>, SnapshotError> {
let saved = match read(path) {
Ok(saved) => saved,
Err(SnapshotError::Io(err)) if err.kind() == std::io::ErrorKind::NotFound => {
return Ok(Vec::new());
}
Err(err) => return Err(err),
};
let restorable = restorable(saved);
for (name, err) in &restorable.rejected {
tracing::warn!(name, %err, "muster roll entry rejected on restore");
}
if restorable.members.is_empty() {
return Ok(Vec::new());
}
let restored: Vec<String> = restorable
.members
.iter()
.map(|app| app.config().name.clone())
.collect();
let running = supervisor.list_checked().await.unwrap_or_default();
let known = |app: &ResolvedApp| running.iter().any(|info| info.name == app.config().name);
warn_about_dogs_holding_sheep_names(&restorable.members, &running);
let starting: Vec<&str> = restorable
.to_start
.iter()
.map(|app| app.config().name.as_str())
.collect();
let members: Vec<ResolvedApp> = restorable
.members
.iter()
.filter(|app| !known(app) && !starting.contains(&app.config().name.as_str()))
.cloned()
.collect();
if !members.is_empty() {
registry.record(&members);
if let Err(err) = supervisor
.register_at_rest(members, restorable.disabled.clone())
.await
{
tracing::warn!(%err, "muster roll restore could not register one or more apps");
}
}
let to_start: Vec<ResolvedApp> = restorable
.to_start
.into_iter()
.filter(|app| !known(app))
.collect();
if to_start.is_empty() {
return Ok(restored);
}
registry.record(&to_start);
let plan = crate::boot_order::plan_for(&to_start, dogs, boot_first_dogs);
warn_about_the_graph(
&plan,
&to_start,
&restorable.members,
&running,
dogs,
boot_first_dogs,
);
warn_about_disabled_dependencies(&to_start, &restorable.disabled);
if let Err(err) = crate::boot_order::start_in_stages(
&plan,
&to_start,
supervisor,
events,
BatchPolicy::PerApp,
)
.await
{
tracing::warn!(%err, "muster roll restore could not start one or more apps");
}
Ok(restored)
}
fn warn_about_disabled_dependencies(to_start: &[ResolvedApp], disabled: &BTreeSet<String>) {
for app in to_start {
for target in &app.config().depends_on {
if disabled.contains(target) {
tracing::warn!(
sheep = %app.config().name,
dependency = %target,
"the dependency is disabled; starting without it"
);
}
}
}
}
fn warn_about_dogs_holding_sheep_names(members: &[ResolvedApp], running: &[ProcessInfo]) {
for app in members {
let name = app.config().name.as_str();
let Some(source) = running
.iter()
.find(|info| info.name == name)
.and_then(|info| info.dog.as_ref())
else {
continue;
};
tracing::warn!(
name,
dog = ?source,
"a dog holds this name, so the roll's sheep of that name is not restored"
);
}
}
fn is_up(running: &[ProcessInfo], name: &str) -> bool {
running.iter().any(|info| {
info.name == name && !matches!(info.status, ProcStatus::Stopped | ProcStatus::Errored)
})
}
fn warn_about_the_graph(
plan: &BootPlan,
to_start: &[ResolvedApp],
members: &[ResolvedApp],
running: &[ProcessInfo],
dogs: &[String],
boot_first_dogs: &[String],
) {
let opted_out: BTreeSet<&str> = members
.iter()
.filter(|app| !app.config().autostart)
.map(|app| app.config().name.as_str())
.collect();
let restorable: BTreeSet<&str> = members
.iter()
.map(|app| app.config().name.as_str())
.collect();
for unresolved in &plan.unresolved {
if restorable.contains(unresolved.missing.as_str()) {
continue;
}
tracing::warn!(
sheep = %unresolved.dependent,
missing = %unresolved.missing,
"a dependency names nothing this flock has; starting without it"
);
}
for app in to_start {
for target in &app.config().depends_on {
if opted_out.contains(target.as_str()) {
tracing::warn!(
sheep = %app.config().name,
dependency = %target,
"the dependency sets autostart = false; starting without it"
);
}
}
}
let unpromoted: BTreeSet<&str> = dogs
.iter()
.map(String::as_str)
.filter(|dog| !boot_first_dogs.iter().any(|first| first == dog))
.filter(|dog| !is_up(running, dog))
.filter(|dog| !to_start.iter().any(|app| app.config().name == *dog))
.collect();
for app in to_start {
for target in &app.config().depends_on {
if unpromoted.contains(target.as_str()) {
tracing::warn!(
sheep = %app.config().name,
dog = %target,
"the dependency is a dog outside [daemon] boot_first_dogs, \
which starts after the whole flock; starting without it"
);
}
}
}
for cycle in &plan.cycles {
tracing::warn!(
cycle = %render_cycle(cycle),
"a dependency cycle; those sheep start last, in no particular order"
);
}
if let Some(stuck) = cyclic_stage(plan) {
let named: BTreeSet<&str> = plan.cycles.iter().flatten().map(String::as_str).collect();
if stuck.iter().any(|name| !named.contains(name.as_str())) {
let names: Vec<&str> = stuck.iter().map(String::as_str).collect();
tracing::warn!(
stuck = ?names,
"every sheep a dependency cycle holds; each of them starts last"
);
}
}
}
fn cyclic_stage(plan: &BootPlan) -> Option<&Vec<String>> {
let member = plan.cycles.first()?.first()?;
plan.stages.iter().find(|stage| stage.contains(member))
}
fn is_state_change(event: &BusEvent) -> bool {
matches!(event, BusEvent::Process { .. })
}
#[derive(Debug)]
pub(crate) struct SnapshotWriter {
handle: JoinHandle<()>,
#[allow(dead_code, reason = "read by this crate's own tests through `writes`")]
writes: Arc<AtomicU64>,
}
impl SnapshotWriter {
#[inline]
#[must_use]
#[allow(dead_code, reason = "called by this module's own tests")]
pub(crate) fn writes(&self) -> u64 {
self.writes.load(Ordering::SeqCst)
}
pub(crate) async fn stop(self) {
self.handle.abort();
let _ = self.handle.await;
}
}
pub(crate) fn spawn_snapshot_writer(
path: PathBuf,
supervisor: SupervisorHandle,
registry: FlockRegistry,
events: broadcast::Receiver<SharedEvent>,
) -> SnapshotWriter {
let writes = Arc::new(AtomicU64::new(0));
let task_writes = Arc::clone(&writes);
let dirty = Arc::clone(®istry.dirty);
let handle = tokio::spawn(run_writer(
path,
supervisor,
registry,
events,
dirty,
task_writes,
));
SnapshotWriter { handle, writes }
}
async fn run_writer(
path: PathBuf,
supervisor: SupervisorHandle,
registry: FlockRegistry,
mut events: broadcast::Receiver<SharedEvent>,
dirty: Arc<Notify>,
writes: Arc<AtomicU64>,
) {
let mut deadline: Option<Instant> = None;
loop {
tokio::select! {
received = events.recv() => match received {
Ok(event) => if is_state_change(&event) && deadline.is_none() {
deadline = Some(Instant::now() + Duration::from_millis(SNAPSHOT_DEBOUNCE_MS));
},
Err(RecvError::Lagged(_)) => if deadline.is_none() {
deadline = Some(Instant::now() + Duration::from_millis(SNAPSHOT_DEBOUNCE_MS));
},
Err(RecvError::Closed) => break,
},
() = dirty.notified() => if deadline.is_none() {
deadline = Some(Instant::now() + Duration::from_millis(SNAPSHOT_DEBOUNCE_MS));
},
() = sleep_until(deadline.unwrap_or_else(Instant::now)), if deadline.is_some() => {
deadline = None;
write_now(&path, &supervisor, ®istry, &writes).await;
}
}
}
}
async fn write_now(
path: &Path,
supervisor: &SupervisorHandle,
registry: &FlockRegistry,
writes: &AtomicU64,
) {
let Ok(infos) = supervisor.list_checked().await else {
return;
};
let roll = registry.roll(&infos, crate::now_ms()); match write_atomic(path, &roll) {
Ok(()) => {
writes.fetch_add(1, Ordering::SeqCst);
}
Err(err) => tracing::warn!(%err, "muster roll write failed"),
}
}
#[cfg(test)]
mod tests {
use super::*;
use shep_core::config::AppConfig;
use shep_core::protocol::ProcessInfo;
use shep_core::status::ProcStatus;
fn info(id: u32, name: &str, status: ProcStatus) -> ProcessInfo {
ProcessInfo::builder(id, name, status)
.pid(Some(1000 + id))
.out_file(Some(format!("/logs/{name}-0-out.log")))
.err_file(Some(format!("/logs/{name}-0-err.log")))
.build()
}
mod disabled;
mod graph;
mod io;
mod muster;
mod roll;
mod secrets;
mod writer;
#[test]
fn a_sheep_saved_while_stopped_is_still_a_member() {
let roll = FlockSnapshot::with_apps(vec![SavedApp {
app: AppConfig::minimal("api-auth", "./api-auth"),
instances_running: 0,
disabled: false,
}]);
let restorable = restorable(roll);
assert_eq!(restorable.members.len(), 1, "stopping is not forgetting");
assert_eq!(restorable.members[0].config().name, "api-auth");
assert!(
restorable.to_start.is_empty(),
"but it stays stopped: it was not running when the roll was written"
);
}
}