use super::filter::{RootedFilter, WatchFilter};
use crate::supervisor::{SupervisorError, SupervisorHandle};
use crate::watch::source::{WatchBatch, WatchError, watch_tree};
use core::time::Duration;
use shep_core::selector::ProcessSelector;
use std::path::PathBuf;
use tokio::sync::mpsc;
pub(crate) const DEFAULT_WATCH_DELAY: Duration = Duration::from_millis(500);
pub(crate) const MIN_WATCH_DELAY: Duration = Duration::from_millis(1);
pub fn spawn_watch_group(
name: String,
root: PathBuf,
filter: WatchFilter,
delay: Duration,
supervisor: SupervisorHandle,
) -> Result<tokio::task::JoinHandle<()>, WatchError> {
let (source, rx) = watch_tree(&root, delay)?;
let filter = RootedFilter { root, filter };
let handle = tokio::spawn(async move {
let _source = source;
run_group(name, filter, rx, supervisor).await;
});
Ok(handle)
}
pub(super) async fn run_group(
name: String,
filter: RootedFilter,
mut rx: mpsc::UnboundedReceiver<WatchBatch>,
supervisor: SupervisorHandle,
) {
loop {
let Some(mut batch) = rx.recv().await else {
return; };
while let Ok(more) = rx.try_recv() {
batch.paths.extend(more.paths);
batch.rescan |= more.rescan;
}
if !batch.rescan && !batch.paths.iter().any(|path| filter.triggers(path)) {
continue;
}
match supervisor
.restart_automatic(ProcessSelector::Name(name.clone()))
.await
{
Ok(_) => {}
Err(SupervisorError::NotFound) => {
tracing::debug!(name, "watch fired but no sheep by this name is registered");
}
Err(err @ SupervisorError::SpawnFailed(_)) => {
tracing::warn!(name, %err, "watch-triggered restart failed to spawn");
}
Err(
err @ (SupervisorError::ReopenFailed(_)
| SupervisorError::FlushFailed(_)
| SupervisorError::ReloadInFlight(_)
| SupervisorError::InvalidScale(_)
| SupervisorError::CannotStart(_)
| SupervisorError::IsADog(_)
| SupervisorError::Disabled(_)
| SupervisorError::InvalidEnv(_)
| SupervisorError::InvalidField(_)
| SupervisorError::Overrides(_)
| SupervisorError::NoSuchSheep(_)
| SupervisorError::EnvChanged(_)),
) => {
tracing::warn!(name, %err, "watch-triggered restart reported an unrelated failure");
}
Err(err @ SupervisorError::EngineStopped) => {
tracing::warn!(name, %err, "supervisor engine has shut down; watch worker ending");
return;
}
}
}
}
#[cfg(test)]
mod tests {
use super::super::filter::{RootedFilter, WatchFilter};
use crate::watch::real_time;
use core::time::Duration;
use std::path::PathBuf;
use super::*;
use crate::supervisor::SupervisorError;
use crate::watch::source::WatchError;
use shep_core::selector::ProcessSelector;
use tokio::sync::broadcast;
use tokio::sync::mpsc;
use super::super::testing::*;
use crate::fake::{ProcScript, ScriptedRunner};
use crate::supervisor::spawn_supervisor;
use crate::testing::test_paths;
use shep_core::config::{AppConfig, normalize};
use shep_core::protocol::{BusEvent, ProcessEventKind};
use shep_core::status::ProcStatus;
use shep_core::values::UpDuration;
#[tokio::test(start_paused = true)]
async fn a_batch_with_one_triggering_path_produces_exactly_one_restart() {
let (handle, mut rx, _dir) = spawn_test_fixture(vec![ProcScript::never_exits(); 3]);
let name = "web";
start_app(&handle, name, 1).await;
let root = PathBuf::from("/watched");
let (tx, group) = spawn_group_matching_everything(&root, name, &handle);
tx.send(changed(vec![root.join("src/main.rs")])).unwrap();
let info = expect_restart(&mut rx, name, EVENT_WAIT).await;
assert_eq!(info.restarts, 1);
assert_no_restart_within(&mut rx, name, Duration::from_secs(5)).await;
group.abort();
}
#[tokio::test(start_paused = true)]
async fn a_rescan_restarts_under_a_non_matching_watch_options() {
let (handle, mut rx, _dir) = spawn_test_fixture(vec![ProcScript::never_exits(); 2]);
let name = "web";
start_app(&handle, name, 1).await;
let root = PathBuf::from("/watched");
let filter = RootedFilter {
root,
filter: WatchFilter::new(&["src/**/*.rs".to_string()], &[]).unwrap(),
};
let (tx, group_rx) = mpsc::unbounded_channel();
let group = tokio::spawn(run_group(
name.to_string(),
filter,
group_rx,
handle.clone(),
));
tx.send(rescan_marker()).unwrap();
let info = expect_restart(&mut rx, name, EVENT_WAIT).await;
assert_eq!(info.restarts, 1);
group.abort();
}
#[tokio::test(start_paused = true)]
async fn an_ordinary_event_on_the_root_itself_produces_no_restart() {
let (handle, mut rx, _dir) = spawn_test_fixture(vec![ProcScript::never_exits(); 2]);
let name = "web";
start_app(&handle, name, 1).await;
let root = PathBuf::from("/watched");
let (tx, group) = spawn_group_matching_everything(&root, name, &handle);
tx.send(changed(vec![root.clone()])).unwrap();
assert_no_restart_within(&mut rx, name, Duration::from_secs(5)).await;
tx.send(changed(vec![root.join("src/main.rs")])).unwrap();
let info = expect_restart(&mut rx, name, EVENT_WAIT).await;
assert_eq!(info.restarts, 1);
group.abort();
}
#[tokio::test(start_paused = true)]
async fn a_rescan_queued_behind_an_ignored_batch_survives_the_drain() {
let (handle, mut rx, _dir) = spawn_test_fixture(vec![ProcScript::never_exits(); 2]);
let name = "web";
start_app(&handle, name, 1).await;
let root = PathBuf::from("/watched");
let (tx, group) = spawn_group_matching_everything(&root, name, &handle);
tx.send(changed(vec![root.join(".git/index")])).unwrap();
tx.send(rescan_marker()).unwrap();
let info = expect_restart(&mut rx, name, EVENT_WAIT).await;
assert_eq!(info.restarts, 1);
group.abort();
}
#[tokio::test(start_paused = true)]
async fn a_batch_queued_during_a_restart_is_rechecked_and_drained_as_one() {
let (handle, mut rx, _dir) = spawn_test_fixture(vec![ProcScript::ignores_signals(); 4]);
let name = "web";
start_app(&handle, name, 1).await;
let root = PathBuf::from("/watched");
let (tx, group) = spawn_group_matching_everything(&root, name, &handle);
tx.send(changed(vec![root.join("a.rs")])).unwrap();
settle().await;
tx.send(changed(vec![root.join("b.rs")])).unwrap();
tx.send(changed(vec![root.join("c.rs")])).unwrap();
let first = expect_restart(&mut rx, name, EVENT_WAIT).await;
assert_eq!(first.restarts, 1);
let second = expect_restart(&mut rx, name, EVENT_WAIT).await;
assert_eq!(second.restarts, 2);
assert_no_restart_within(&mut rx, name, Duration::from_secs(5)).await;
group.abort();
}
#[tokio::test(start_paused = true)]
async fn dropping_the_sender_ends_the_group_task() {
let (handle, _rx, _dir) = spawn_test_fixture(vec![]);
let root = PathBuf::from("/watched");
let (tx, group) = spawn_group_matching_everything(&root, "ghost", &handle);
drop(tx);
tokio::time::timeout(EVENT_WAIT, group)
.await
.expect("group task did not end after its sender was dropped")
.expect("group task panicked");
}
#[tokio::test(start_paused = true)]
async fn a_triggering_batch_restarts_a_stopped_instance_in_the_same_group() {
let (handle, mut rx, _dir) = spawn_test_fixture(vec![ProcScript::never_exits(); 4]);
let name = "web";
let infos = start_app(&handle, name, 2).await;
let stopped_id = infos[1].id;
handle.stop(ProcessSelector::Id(stopped_id)).await.unwrap();
let root = PathBuf::from("/watched");
let (tx, group) = spawn_group_matching_everything(&root, name, &handle);
tx.send(changed(vec![root.join("src/main.rs")])).unwrap();
let first = expect_restart(&mut rx, name, EVENT_WAIT).await;
let second = expect_restart(&mut rx, name, EVENT_WAIT).await;
let stopped_info = [first, second]
.into_iter()
.find(|info| info.id == stopped_id)
.expect("the previously-stopped instance never restarted");
assert_eq!(stopped_info.status, ProcStatus::Online);
assert_eq!(stopped_info.restarts, 1);
group.abort();
}
#[tokio::test(start_paused = true)]
async fn an_operators_stop_beats_a_watch_triggered_restart_mid_ladder() {
let (handle, mut rx, _dir) = spawn_test_fixture(vec![
ProcScript::ignores_signals(), ProcScript::never_exits(), ProcScript::never_exits(), ProcScript::never_exits(), ]);
let name = "web";
let infos = start_app(&handle, name, 2).await;
let (held, released) = (infos[0].id, infos[1].id);
let root = PathBuf::from("/watched");
let (tx, group) = spawn_group_matching_everything(&root, name, &handle);
tx.send(changed(vec![root.join("src/main.rs")])).unwrap();
let restarted = expect_restart(&mut rx, name, EVENT_WAIT).await;
assert_eq!(
(restarted.id, restarted.restarts),
(released, 1),
"the batch never reached the actor, so the stop below would race \
nothing -- got {restarted:?}"
);
group.abort();
let stopped = handle.stop(ProcessSelector::Id(held)).await.unwrap();
assert_eq!(stopped.len(), 1);
assert_eq!(
(stopped[0].id, stopped[0].status, stopped[0].restarts),
(held, ProcStatus::Stopped, 0),
"an operator's stop was silently converted into the watch-triggered \
restart it raced -- got {stopped:?}"
);
let listed = handle.list().await;
assert_eq!(
(listed[0].id, listed[0].status, listed[0].pid),
(held, ProcStatus::Stopped, None),
"the sheep an operator stopped is running again -- got {listed:?}"
);
assert_eq!(
(listed[1].id, listed[1].status),
(released, ProcStatus::Online),
"the instance the operator did not name must still be up, \
restarted by the batch -- got {listed:?}"
);
}
#[tokio::test(start_paused = true)]
async fn not_found_leaves_the_loop_alive_for_the_next_batch() {
let (handle, mut rx, _dir) = spawn_test_fixture(vec![ProcScript::never_exits(); 2]);
let name = "ghost";
let root = PathBuf::from("/watched");
let (tx, group) = spawn_group_matching_everything(&root, name, &handle);
tx.send(changed(vec![root.join("a.rs")])).unwrap();
assert_no_restart_within(&mut rx, name, Duration::from_millis(200)).await;
assert!(!group.is_finished(), "the loop must not exit on NotFound");
start_app(&handle, name, 1).await;
tx.send(changed(vec![root.join("b.rs")])).unwrap();
let info = expect_restart(&mut rx, name, EVENT_WAIT).await;
assert_eq!(info.restarts, 1);
group.abort();
}
#[tokio::test(start_paused = true)]
async fn the_group_task_ends_when_the_supervisor_engine_has_stopped() {
let (handle, _rx, _dir) = spawn_test_fixture(Vec::new());
let name = "web";
handle.shutdown().await;
assert_eq!(
handle
.restart_automatic(ProcessSelector::Name(name.to_string()))
.await
.unwrap_err(),
SupervisorError::EngineStopped
);
let root = PathBuf::from("/watched");
let (tx, group) = spawn_group_matching_everything(&root, name, &handle);
tx.send(changed(vec![root.join("src/main.rs")])).unwrap();
tokio::time::timeout(EVENT_WAIT, group)
.await
.expect("the group task did not end after the engine shut down")
.expect("group task panicked");
drop(tx); }
#[tokio::test]
async fn a_watch_root_that_does_not_exist_names_the_path_it_could_not_watch() {
let (handle, _rx, _dir) = spawn_test_fixture(vec![]);
let parent = tempfile::tempdir().unwrap();
let missing = parent.path().join("no-such-directory");
let err = spawn_watch_group(
"web".to_string(),
missing.clone(),
WatchFilter::new(&[], &[]).unwrap(),
real_time::TEST_DELAY,
handle,
)
.unwrap_err();
let WatchError::Watch { path, reason } = err else {
panic!("a root that does not exist must report `Watch`, got {err:?}");
};
assert_eq!(path, missing, "`Watch` must carry the root it was handed");
assert!(!reason.is_empty(), "`Watch` must carry notify's own reason");
}
proptest::proptest! {
#![proptest_config(crate::testing::proptest_config(64))]
#[test]
fn a_watch_group_never_has_two_restarts_in_flight(
batches in proptest::collection::vec(batch_strategy(), 1..=MAX_BATCHES),
kill_timeout_ms in 200u64..2_000u64,
) {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.start_paused(true)
.build()
.unwrap();
let dir = tempfile::tempdir().unwrap();
runtime.block_on(async move {
let kill_timeout = Duration::from_millis(kill_timeout_ms);
let (events, mut rx) = crate::bus::test_bus(1024);
let runner =
ScriptedRunner::new(vec![ProcScript::ignores_signals(); SINGLE_FLIGHT_SCRIPTS]);
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let name = "web";
let mut app = AppConfig::minimal(name, "./srv");
app.kill_timeout = UpDuration::from_millis(kill_timeout_ms);
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
let root = PathBuf::from("/watched");
let (tx, group) = spawn_group_matching_everything(&root, name, &handle);
let start = tokio::time::Instant::now();
let watched = name.to_string();
let collector = tokio::spawn(async move {
let mut observed = Vec::new();
loop {
match tokio::time::timeout(EVENT_WAIT, rx.recv()).await.map(|received| received.map(|event| event.to_event())) {
Ok(Ok(BusEvent::Process {
event: ProcessEventKind::Restart,
info,
..
})) if info.name == watched => {
observed
.push((tokio::time::Instant::now() - start, info.restarts));
}
Ok(Ok(_)) => continue,
Ok(Err(broadcast::error::RecvError::Lagged(skipped))) => {
return Err(skipped);
}
Ok(Err(broadcast::error::RecvError::Closed)) => break,
Err(_elapsed) => break, }
}
Ok(observed)
});
for (i, batch) in batches.iter().enumerate() {
if batch.gap > Duration::ZERO {
tokio::time::sleep(batch.gap).await;
}
let path = if batch.triggers {
root.join(format!("src/f{i}.rs"))
} else {
root.join(format!(".git/o{i}"))
};
tx.send(changed(vec![path])).unwrap();
}
let observed = match collector.await.expect("collector task panicked") {
Ok(observed) => observed,
Err(skipped) => {
return Err(proptest::test_runner::TestCaseError::fail(format!(
"event stream lagged by {skipped}"
)));
}
};
group.abort();
for pair in observed.windows(2) {
proptest::prop_assert!(
pair[1].0 - pair[0].0 >= kill_timeout,
"two restarts of {} finished {:?} apart, less than the {:?} one takes: \
they overlapped",
name,
pair[1].0 - pair[0].0,
kill_timeout
);
}
let expected = expected_restart_instants(&batches, kill_timeout);
let counted: Vec<u32> = (1..=expected.len() as u32).collect();
proptest::prop_assert_eq!(
observed,
expected.into_iter().zip(counted).collect::<Vec<_>>()
);
Ok::<(), proptest::test_runner::TestCaseError>(())
})?;
}
}
mod slow {
use super::*;
#[tokio::test]
async fn spawn_watch_group_restarts_on_a_real_touch_and_stops_on_abort() {
let (handle, mut rx, _dir) = spawn_test_fixture(vec![ProcScript::never_exits(); 2]);
let name = "web";
start_app(&handle, name, 1).await;
let watch_dir = tempfile::tempdir().unwrap();
let root = watch_dir.path().canonicalize().unwrap();
let filter = WatchFilter::new(&[], &[]).unwrap();
let group = spawn_watch_group(
name.to_string(),
root.clone(),
filter,
real_time::TEST_DELAY,
handle.clone(),
)
.unwrap();
crate::testing::touch(&root, "trigger.txt").unwrap();
let info = expect_restart(&mut rx, name, real_time::SMOKE_DEADLINE).await;
assert_eq!(info.restarts, 1);
group.abort();
crate::testing::touch(&root, "after-abort.txt").unwrap();
assert_no_restart_within(&mut rx, name, real_time::NO_EVENT_WINDOW).await;
}
}
}