use core::fmt;
use core::time::Duration;
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use tokio::sync::mpsc;
use tokio::task::JoinHandle;
use shep_core::config::{AppConfig, CronSchedule, ProbeTarget};
use shep_core::values::UpDuration;
use crate::cron::{Clock, SystemClock, spawn_cron_worker};
use crate::entry::ProcessEntry;
use crate::limits::sample::{MemorySampler, SysinfoSampler};
use crate::limits::stats::StatsState;
use crate::limits::{LimitBreach, LimitEnforcer, PollingEnforcer};
use crate::probes::{LivenessFailure, Prober, spawn_liveness_task};
use crate::supervisor::SupervisorHandle;
use crate::watch::{
DEFAULT_WATCH_DELAY, MIN_WATCH_DELAY, WatchFilter, own_log_ignores, spawn_watch_group,
};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct LivenessReport {
pub id: u32,
pub pid: u32,
pub epoch: u64,
}
#[derive(Debug, Clone)]
pub struct ExtrasReports {
pub breaches: mpsc::Sender<LimitBreach>,
pub liveness: mpsc::Sender<LivenessReport>,
}
pub struct Extras {
pub clock: Arc<dyn Clock>,
pub enforcer: Arc<dyn LimitEnforcer>,
pub max_cron_sleep: Duration,
pub reports: ExtrasReports,
pub stats: Arc<StatsState>,
}
impl Extras {
#[must_use]
pub fn real(reports: ExtrasReports, max_cron_sleep: Duration) -> Self {
let sampler: Arc<dyn MemorySampler> = Arc::new(SysinfoSampler::new());
let stats = Arc::new(StatsState::new(Arc::clone(&sampler)));
let enforcer =
PollingEnforcer::start(sampler, reports.breaches.clone(), Arc::clone(&stats));
Self {
clock: Arc::new(SystemClock),
enforcer: Arc::new(enforcer),
max_cron_sleep,
reports,
stats,
}
}
}
impl fmt::Debug for Extras {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Extras")
.field("clock", &"<dyn Clock>")
.field("enforcer", &"<dyn LimitEnforcer>")
.field("max_cron_sleep", &self.max_cron_sleep)
.finish_non_exhaustive()
}
}
pub fn spawn_extras_reporter(
mut breaches: mpsc::Receiver<LimitBreach>,
mut liveness: mpsc::Receiver<LivenessReport>,
supervisor: SupervisorHandle,
) -> JoinHandle<()> {
tokio::spawn(async move {
let mut breaches_open = true;
let mut liveness_open = true;
while breaches_open || liveness_open {
tokio::select! {
maybe_breach = breaches.recv(), if breaches_open => match maybe_breach {
Some(breach) => {
tracing::warn!(
id = breach.id,
pid = breach.root_pid,
observed = %breach.observed,
limit = %breach.limit,
"process tree exceeded its max_memory; restarting"
);
supervisor
.extra_restart(
breach.id,
breach.root_pid,
None,
Some(breach.observed),
)
.await;
}
None => breaches_open = false,
},
maybe_failure = liveness.recv(), if liveness_open => match maybe_failure {
Some(report) => {
tracing::warn!(
id = report.id,
pid = report.pid,
"liveness probe hit its failure_threshold; restarting"
);
supervisor
.extra_restart(report.id, report.pid, Some(report.epoch), None)
.await;
}
None => liveness_open = false,
},
}
}
})
}
#[derive(Debug, Default)]
pub struct ExtrasRegistry {
groups: HashMap<String, NameExtras>,
instances: HashMap<u32, InstanceExtras>,
liveness_epochs: HashMap<u32, u64>,
}
#[derive(Debug, Default)]
struct NameExtras {
cron: Option<JoinHandle<()>>,
watch: Option<JoinHandle<()>>,
members: HashSet<u32>,
}
impl NameExtras {
fn abort(self) {
if let Some(cron) = self.cron {
cron.abort();
}
if let Some(watch) = self.watch {
watch.abort();
}
}
}
struct InstanceExtras {
stats: Arc<StatsState>,
limit: Option<Arc<dyn LimitEnforcer>>,
liveness: Option<JoinHandle<()>>,
}
impl InstanceExtras {
fn watched_only(stats: Arc<StatsState>) -> Self {
Self {
stats,
limit: None,
liveness: None,
}
}
fn disarm(self, id: u32) {
self.stats.unwatch(id);
if let Some(enforcer) = self.limit {
enforcer.disarm(id);
}
if let Some(liveness) = self.liveness {
liveness.abort();
}
}
}
impl fmt::Debug for InstanceExtras {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("InstanceExtras")
.field("limit_armed", &self.limit.is_some())
.field("liveness", &self.liveness)
.finish_non_exhaustive()
}
}
impl ExtrasRegistry {
pub fn arm(
&mut self,
entry: &ProcessEntry,
prober: Arc<dyn Prober>,
extras: &Extras,
supervisor: &SupervisorHandle,
) {
let config = entry.spec.config();
let id = entry.id;
self.disarm_instance(id);
let liveness_epoch = self.liveness_epochs.entry(id).or_insert(0);
*liveness_epoch += 1;
let liveness_epoch = *liveness_epoch;
if let Some(instance) = arm_instance(entry, prober, extras, liveness_epoch) {
self.instances.insert(id, instance);
}
if config.cron_restart.is_none() && !config.watch {
return;
}
let group = self.groups.entry(config.name.clone()).or_default();
group.members.insert(id);
if group.cron.as_ref().is_none_or(JoinHandle::is_finished) {
group.cron = arm_cron(config, extras, supervisor);
}
if group.watch.as_ref().is_none_or(JoinHandle::is_finished) {
group.watch = arm_watch(entry, supervisor);
}
}
pub fn disarm(&mut self, id: u32, name: &str) {
self.disarm_instance(id);
self.liveness_epochs.remove(&id);
let Some(group) = self.groups.get_mut(name) else {
return;
};
if !group.members.remove(&id) || !group.members.is_empty() {
return;
}
if let Some(group) = self.groups.remove(name) {
group.abort();
}
}
pub fn rearm_name(
&mut self,
name: &str,
entries: &[&ProcessEntry],
prober: impl Fn(&ProcessEntry) -> Arc<dyn Prober>,
extras: &Extras,
supervisor: &SupervisorHandle,
) {
if let Some(group) = self.groups.remove(name) {
group.abort();
}
for entry in entries {
self.arm(entry, prober(entry), extras, supervisor);
}
}
fn disarm_instance(&mut self, id: u32) {
if let Some(instance) = self.instances.remove(&id) {
instance.disarm(id);
}
}
pub(crate) fn liveness_epoch(&self, id: u32) -> u64 {
self.liveness_epochs.get(&id).copied().unwrap_or(0)
}
#[cfg(test)]
pub(crate) fn group_members(&self, name: &str) -> Option<Vec<u32>> {
self.groups.get(name).map(|group| {
let mut members: Vec<u32> = group.members.iter().copied().collect();
members.sort_unstable();
members
})
}
}
impl Drop for ExtrasRegistry {
fn drop(&mut self) {
for (id, instance) in self.instances.drain() {
instance.disarm(id);
}
for (_name, group) in self.groups.drain() {
group.abort();
}
}
}
fn arm_instance(
entry: &ProcessEntry,
prober: Arc<dyn Prober>,
extras: &Extras,
liveness_epoch: u64,
) -> Option<InstanceExtras> {
let config = entry.spec.config();
let wants_anything = config.max_memory.is_some() || config.liveness_probe.is_some();
let Some(pid) = entry.pid else {
if wants_anything {
tracing::warn!(
id = entry.id,
"arming a sheep with no pid; its memory limit and liveness probe stay disarmed"
);
}
return None;
};
extras.stats.watch(entry.id, pid);
let mut instance = InstanceExtras::watched_only(Arc::clone(&extras.stats));
if let Some(limit) = config.max_memory {
extras.enforcer.arm(entry.id, pid, limit);
instance.limit = Some(Arc::clone(&extras.enforcer));
}
if let Some(probe) = config.liveness_probe.as_ref() {
match ProbeTarget::parse(probe) {
Ok(target) => {
let (raw_tx, mut raw_rx) = mpsc::channel::<LivenessFailure>(1);
let reports_liveness = extras.reports.liveness.clone();
tokio::spawn(async move {
if let Some(failure) = raw_rx.recv().await {
let _ = reports_liveness
.send(LivenessReport {
id: failure.id,
pid: failure.pid,
epoch: liveness_epoch,
})
.await;
}
});
instance.liveness = Some(spawn_liveness_task(
entry.id,
pid,
probe.clone(),
target,
prober,
raw_tx,
));
}
Err(err) => {
tracing::warn!(
id = entry.id,
name = config.name.as_str(),
%err,
"liveness_probe target could not be parsed; arming no liveness probe"
);
}
}
}
Some(instance)
}
fn arm_cron(
config: &AppConfig,
extras: &Extras,
supervisor: &SupervisorHandle,
) -> Option<JoinHandle<()>> {
let pattern = config.cron_restart.as_ref()?;
let schedule = match CronSchedule::parse(pattern, config.cron_timezone.as_deref()) {
Ok(schedule) => schedule,
Err(err) => {
tracing::warn!(
name = config.name.as_str(),
pattern = pattern.as_str(),
%err,
"cron_restart pattern could not be parsed; arming no cron worker"
);
return None;
}
};
Some(spawn_cron_worker(
config.name.clone(),
schedule,
Arc::clone(&extras.clock),
supervisor.clone(),
extras.max_cron_sleep,
))
}
fn arm_watch(entry: &ProcessEntry, supervisor: &SupervisorHandle) -> Option<JoinHandle<()>> {
let config = entry.spec.config();
if !config.watch {
return None;
}
let Some(cwd) = config.cwd.as_deref() else {
tracing::warn!(
name = config.name.as_str(),
"watch is on but the app names no cwd; arming no watch"
);
return None;
};
let root = match std::fs::canonicalize(cwd) {
Ok(root) => root,
Err(err) => {
tracing::warn!(
name = config.name.as_str(),
path = cwd,
%err,
"watch root could not be resolved; arming no watch"
);
return None;
}
};
let mut ignores = config.ignore_watch.clone();
ignores.extend(own_log_ignores(
&root,
[entry.out_file.as_path(), entry.err_file.as_path()],
));
let filter = match WatchFilter::new(&config.watch_options, &ignores) {
Ok(filter) => filter,
Err(err) => {
tracing::warn!(
name = config.name.as_str(),
%err,
"watch globs could not be compiled; arming no watch"
);
return None;
}
};
match spawn_watch_group(
config.name.clone(),
root,
filter,
watch_delay_for(config),
supervisor.clone(),
) {
Ok(handle) => Some(handle),
Err(err) => {
tracing::warn!(
name = config.name.as_str(),
%err,
"the OS watch could not be started; arming no watch"
);
None
}
}
}
fn watch_delay_for(config: &AppConfig) -> Duration {
config
.watch_delay
.map(UpDuration::as_duration)
.unwrap_or(DEFAULT_WATCH_DELAY)
.max(MIN_WATCH_DELAY)
}
#[cfg(test)]
mod tests;