pub mod source;
use core::fmt;
use core::time::Duration;
use std::path::{Path, PathBuf};
use globset::{Glob, GlobSet, GlobSetBuilder};
use tokio::sync::mpsc;
use shep_core::selector::ProcessSelector;
use crate::supervisor::{SupervisorError, SupervisorHandle};
use crate::watch::source::{WatchBatch, WatchError, watch_tree};
pub(crate) const DEFAULT_WATCH_DELAY: Duration = Duration::from_millis(500);
pub(crate) const MIN_WATCH_DELAY: Duration = Duration::from_millis(1);
const DEFAULT_IGNORE_GLOBS: &[&str] = &[
"**/.*",
"**/.*/**",
"**/node_modules/**",
"**/logs/**",
"**/pids/**",
];
const MATCH_EVERYTHING: &str = "**";
pub(crate) fn own_log_ignores<'a>(
root: &Path,
logs: impl IntoIterator<Item = &'a Path>,
) -> Vec<String> {
logs.into_iter()
.filter_map(|log| literal_glob_under(root, log))
.collect()
}
fn literal_glob_under(root: &Path, path: &Path) -> Option<String> {
let relative = canonical_parent_of(path);
let relative = relative.strip_prefix(root).ok()?;
let mut pattern = String::new();
for component in relative.iter() {
if !pattern.is_empty() {
pattern.push('/');
}
pattern.push_str(&globset::escape(component.to_str()?));
}
(!pattern.is_empty()).then_some(pattern)
}
fn canonical_parent_of(path: &Path) -> PathBuf {
let (Some(parent), Some(file)) = (path.parent(), path.file_name()) else {
return path.to_path_buf();
};
std::fs::canonicalize(parent).map_or_else(|_| path.to_path_buf(), |dir| dir.join(file))
}
#[derive(Debug)]
pub struct WatchFilter {
include: GlobSet,
ignore: GlobSet,
}
impl WatchFilter {
pub fn new(
watch_options: &[String],
ignore_watch: &[String],
) -> Result<Self, WatchFilterError> {
let include_patterns: Vec<String> = if watch_options.is_empty() {
vec![MATCH_EVERYTHING.to_string()]
} else {
watch_options.to_vec()
};
let ignore_patterns: Vec<String> = DEFAULT_IGNORE_GLOBS
.iter()
.map(|pattern| (*pattern).to_string())
.chain(ignore_watch.iter().cloned())
.collect();
Ok(Self {
include: build_glob_set(&include_patterns)?,
ignore: build_glob_set(&ignore_patterns)?,
})
}
#[must_use]
pub fn triggers(&self, path: &Path) -> bool {
self.include.is_match(path) && !self.ignore.is_match(path)
}
}
fn build_glob_set(patterns: &[String]) -> Result<GlobSet, WatchFilterError> {
let mut builder = GlobSetBuilder::new();
for pattern in patterns {
let glob = Glob::new(pattern).map_err(|err| WatchFilterError::Glob {
pattern: pattern.clone(),
reason: err.to_string(),
})?;
builder.add(glob);
}
builder.build().map_err(|err| {
WatchFilterError::Glob {
pattern: patterns.join(", "),
reason: err.to_string(),
}
})
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum WatchFilterError {
Glob {
pattern: String,
reason: String,
},
}
impl fmt::Display for WatchFilterError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Glob { pattern, reason } => {
write!(f, "invalid watch pattern `{pattern}`: {reason}")
}
}
}
}
impl core::error::Error for WatchFilterError {}
#[derive(Debug)]
struct RootedFilter {
root: PathBuf,
filter: WatchFilter,
}
impl RootedFilter {
fn triggers(&self, path: &Path) -> bool {
let Ok(relative) = path.strip_prefix(&self.root) else {
return false;
};
if relative.as_os_str().is_empty() {
return false;
}
self.filter.triggers(relative)
}
}
pub fn spawn_watch_group(
name: String,
root: PathBuf,
filter: WatchFilter,
delay: Duration,
supervisor: SupervisorHandle,
) -> Result<tokio::task::JoinHandle<()>, WatchError> {
let (source, rx) = watch_tree(&root, delay)?;
let filter = RootedFilter { root, filter };
let handle = tokio::spawn(async move {
let _source = source;
run_group(name, filter, rx, supervisor).await;
});
Ok(handle)
}
async fn run_group(
name: String,
filter: RootedFilter,
mut rx: mpsc::UnboundedReceiver<WatchBatch>,
supervisor: SupervisorHandle,
) {
loop {
let Some(mut batch) = rx.recv().await else {
return; };
while let Ok(more) = rx.try_recv() {
batch.paths.extend(more.paths);
batch.rescan |= more.rescan;
}
if !batch.rescan && !batch.paths.iter().any(|path| filter.triggers(path)) {
continue;
}
match supervisor
.restart_automatic(ProcessSelector::Name(name.clone()))
.await
{
Ok(_) => {}
Err(SupervisorError::NotFound) => {
tracing::debug!(name, "watch fired but no sheep by this name is registered");
}
Err(err @ SupervisorError::SpawnFailed(_)) => {
tracing::warn!(name, %err, "watch-triggered restart failed to spawn");
}
Err(
err @ (SupervisorError::ReopenFailed(_)
| SupervisorError::FlushFailed(_)
| SupervisorError::ReloadInFlight(_)
| SupervisorError::InvalidScale(_)
| SupervisorError::CannotStart(_)),
) => {
tracing::warn!(name, %err, "watch-triggered restart reported an unrelated failure");
}
Err(err @ SupervisorError::EngineStopped) => {
tracing::warn!(name, %err, "supervisor engine has shut down; watch worker ending");
return;
}
}
}
}
#[cfg(test)]
pub(crate) mod real_time {
use core::time::Duration;
pub(crate) const TEST_DELAY: Duration = Duration::from_millis(50);
pub(crate) const SMOKE_DEADLINE: Duration = Duration::from_secs(5);
pub(crate) const NO_EVENT_WINDOW: Duration = Duration::from_millis(500);
}
#[cfg(test)]
mod tests {
use tokio::sync::broadcast;
use super::*;
use crate::bus::SharedEvent;
use crate::fake::{ProcScript, ScriptedRunner};
use crate::supervisor::spawn_supervisor;
use crate::testing::test_paths;
use shep_core::config::{AppConfig, normalize};
use shep_core::protocol::{BusEvent, ProcessEventKind, ProcessInfo};
use shep_core::status::ProcStatus;
use shep_core::values::UpDuration;
#[test]
fn empty_watch_options_matches_every_path() {
let filter = WatchFilter::new(&[], &[]).unwrap();
assert!(filter.triggers(Path::new("top.txt")));
assert!(filter.triggers(Path::new("src/a/b.rs")));
}
#[test]
fn an_explicit_pattern_matches_its_own_tree_and_nothing_else() {
let filter = WatchFilter::new(&["src/**/*.rs".to_string()], &[]).unwrap();
assert!(filter.triggers(Path::new("src/a/b.rs")));
assert!(!filter.triggers(Path::new("src/a/b.txt")));
assert!(!filter.triggers(Path::new("other/a.rs")));
}
#[test]
fn default_ignores_beat_an_explicit_include() {
let filter = WatchFilter::new(&["**".to_string()], &[]).unwrap();
assert!(!filter.triggers(Path::new(".git/index")));
assert!(!filter.triggers(Path::new("node_modules/x/y.js")));
}
#[test]
fn an_ignore_watch_entry_beats_an_include() {
let filter = WatchFilter::new(&["**".to_string()], &["dist/**".to_string()]).unwrap();
assert!(!filter.triggers(Path::new("dist/bundle.js")));
assert!(filter.triggers(Path::new("src/main.rs")));
}
#[test]
fn a_pattern_matching_nothing_never_triggers() {
let filter = WatchFilter::new(&["nomatch/**/*.foo".to_string()], &[]).unwrap();
assert!(!filter.triggers(Path::new("src/main.rs")));
}
#[test]
fn an_invalid_glob_is_rejected_with_its_pattern() {
let err = WatchFilter::new(&["[".to_string()], &[]).unwrap_err();
let WatchFilterError::Glob { pattern, reason } = err;
assert_eq!(pattern, "[");
assert!(!reason.is_empty());
let _: &dyn core::error::Error = &WatchFilterError::Glob {
pattern: "[".to_string(),
reason: "boom".to_string(),
};
}
#[test]
fn watch_filter_error_display_names_the_pattern_and_its_reason() {
let err = WatchFilterError::Glob {
pattern: "[".to_string(),
reason: "unclosed character class".to_string(),
};
assert_eq!(
err.to_string(),
"invalid watch pattern `[`: unclosed character class"
);
}
#[test]
fn a_path_outside_the_root_never_triggers() {
let filter = RootedFilter {
root: PathBuf::from("/watched"),
filter: WatchFilter::new(&[], &[]).unwrap(),
};
assert!(!filter.triggers(Path::new("/elsewhere/file.rs")));
assert!(filter.triggers(Path::new("/watched/file.rs")));
}
#[test]
fn the_root_itself_never_triggers_however_wide_the_watch_options() {
let filter = matches_everything(PathBuf::from("/watched"));
assert!(!filter.triggers(Path::new("/watched")));
assert!(filter.triggers(Path::new("/watched/other/a.txt")));
}
#[test]
fn own_log_ignores_covers_only_the_paths_under_the_root() {
let root = tempfile::tempdir().unwrap();
let elsewhere = tempfile::tempdir().unwrap();
let canonical = std::fs::canonicalize(root.path()).unwrap();
let inside = root.path().join("app[0].log");
let outside = elsewhere.path().join("web-0-out.log");
let ignores = own_log_ignores(&canonical, [inside.as_path(), outside.as_path()]);
assert_eq!(ignores, vec!["app[[]0[]].log".to_string()]);
let filter = WatchFilter::new(&[], &ignores).unwrap();
assert!(!filter.triggers(Path::new("app[0].log")));
assert!(filter.triggers(Path::new("app0.log")));
assert!(filter.triggers(Path::new("src/main.rs")));
}
const EVENT_WAIT: Duration = Duration::from_secs(30);
const SETTLE_YIELDS: usize = 16;
async fn settle() {
for _ in 0..SETTLE_YIELDS {
tokio::task::yield_now().await;
}
}
fn spawn_test_fixture(
scripts: Vec<ProcScript>,
) -> (
SupervisorHandle,
broadcast::Receiver<SharedEvent>,
tempfile::TempDir,
) {
let dir = tempfile::tempdir().unwrap();
let (events, rx) = crate::bus::test_bus(64);
let runner = ScriptedRunner::new(scripts);
let handle = spawn_supervisor(runner, test_paths(&dir), events);
(handle, rx, dir)
}
async fn start_app(handle: &SupervisorHandle, name: &str, instances: u32) -> Vec<ProcessInfo> {
let mut app = AppConfig::minimal(name, "./srv");
app.instances = instances;
handle.start(vec![normalize(app).unwrap()]).await.unwrap()
}
async fn expect_restart(
rx: &mut broadcast::Receiver<SharedEvent>,
name: &str,
deadline: Duration,
) -> ProcessInfo {
loop {
match tokio::time::timeout(deadline, rx.recv())
.await
.map(|received| received.map(|event| event.to_event()))
{
Ok(Ok(BusEvent::Process {
event: ProcessEventKind::Restart,
info,
..
})) if info.name == name => return info,
Ok(Ok(_)) => continue,
Ok(Err(broadcast::error::RecvError::Lagged(_))) => continue,
Ok(Err(err)) => panic!("event stream closed before a restart of {name}: {err}"),
Err(_) => panic!("timed out waiting for a watch-triggered restart of {name}"),
}
}
}
async fn assert_no_restart_within(
rx: &mut broadcast::Receiver<SharedEvent>,
name: &str,
window: Duration,
) {
let deadline = tokio::time::Instant::now() + window;
loop {
let remaining = deadline.saturating_duration_since(tokio::time::Instant::now());
match tokio::time::timeout(remaining, rx.recv())
.await
.map(|received| received.map(|event| event.to_event()))
{
Err(_) => return, Ok(Ok(BusEvent::Process {
event: ProcessEventKind::Restart,
info,
..
})) if info.name == name => {
panic!(
"unexpected watch-triggered restart of {name} observed (restarts={})",
info.restarts
);
}
Ok(Ok(_)) => continue,
Ok(Err(broadcast::error::RecvError::Lagged(skipped))) => {
panic!(
"event stream lagged by {skipped} while checking for no restart of \
{name}: a skipped event may have been the restart this forbids"
)
}
Ok(Err(err)) => {
panic!("event channel closed while checking for no restart of {name}: {err}")
}
}
}
}
fn matches_everything(root: PathBuf) -> RootedFilter {
RootedFilter {
root,
filter: WatchFilter::new(&[], &[]).unwrap(),
}
}
fn changed(paths: Vec<PathBuf>) -> WatchBatch {
WatchBatch {
paths,
rescan: false,
}
}
fn rescan_marker() -> WatchBatch {
WatchBatch {
paths: Vec::new(),
rescan: true,
}
}
#[tokio::test(start_paused = true)]
async fn a_batch_of_only_ignored_paths_produces_no_restart() {
let (handle, mut rx, _dir) = spawn_test_fixture(vec![ProcScript::never_exits(); 2]);
let name = "web";
start_app(&handle, name, 1).await;
let root = PathBuf::from("/watched");
let (tx, group_rx) = mpsc::unbounded_channel();
let group = tokio::spawn(run_group(
name.to_string(),
matches_everything(root.clone()),
group_rx,
handle.clone(),
));
tx.send(changed(vec![root.join(".git/index")])).unwrap();
assert_no_restart_within(&mut rx, name, Duration::from_secs(5)).await;
group.abort();
}
#[tokio::test(start_paused = true)]
async fn a_batch_with_one_triggering_path_produces_exactly_one_restart() {
let (handle, mut rx, _dir) = spawn_test_fixture(vec![ProcScript::never_exits(); 3]);
let name = "web";
start_app(&handle, name, 1).await;
let root = PathBuf::from("/watched");
let (tx, group_rx) = mpsc::unbounded_channel();
let group = tokio::spawn(run_group(
name.to_string(),
matches_everything(root.clone()),
group_rx,
handle.clone(),
));
tx.send(changed(vec![root.join("src/main.rs")])).unwrap();
let info = expect_restart(&mut rx, name, EVENT_WAIT).await;
assert_eq!(info.restarts, 1);
assert_no_restart_within(&mut rx, name, Duration::from_secs(5)).await;
group.abort();
}
#[tokio::test(start_paused = true)]
async fn a_rescan_restarts_under_a_non_matching_watch_options() {
let (handle, mut rx, _dir) = spawn_test_fixture(vec![ProcScript::never_exits(); 2]);
let name = "web";
start_app(&handle, name, 1).await;
let root = PathBuf::from("/watched");
let filter = RootedFilter {
root,
filter: WatchFilter::new(&["src/**/*.rs".to_string()], &[]).unwrap(),
};
let (tx, group_rx) = mpsc::unbounded_channel();
let group = tokio::spawn(run_group(
name.to_string(),
filter,
group_rx,
handle.clone(),
));
tx.send(rescan_marker()).unwrap();
let info = expect_restart(&mut rx, name, EVENT_WAIT).await;
assert_eq!(info.restarts, 1);
group.abort();
}
#[tokio::test(start_paused = true)]
async fn an_ordinary_event_on_the_root_itself_produces_no_restart() {
let (handle, mut rx, _dir) = spawn_test_fixture(vec![ProcScript::never_exits(); 2]);
let name = "web";
start_app(&handle, name, 1).await;
let root = PathBuf::from("/watched");
let (tx, group_rx) = mpsc::unbounded_channel();
let group = tokio::spawn(run_group(
name.to_string(),
matches_everything(root.clone()),
group_rx,
handle.clone(),
));
tx.send(changed(vec![root.clone()])).unwrap();
assert_no_restart_within(&mut rx, name, Duration::from_secs(5)).await;
tx.send(changed(vec![root.join("src/main.rs")])).unwrap();
let info = expect_restart(&mut rx, name, EVENT_WAIT).await;
assert_eq!(info.restarts, 1);
group.abort();
}
#[tokio::test(start_paused = true)]
async fn a_rescan_queued_behind_an_ignored_batch_survives_the_drain() {
let (handle, mut rx, _dir) = spawn_test_fixture(vec![ProcScript::never_exits(); 2]);
let name = "web";
start_app(&handle, name, 1).await;
let root = PathBuf::from("/watched");
let (tx, group_rx) = mpsc::unbounded_channel();
let group = tokio::spawn(run_group(
name.to_string(),
matches_everything(root.clone()),
group_rx,
handle.clone(),
));
tx.send(changed(vec![root.join(".git/index")])).unwrap();
tx.send(rescan_marker()).unwrap();
let info = expect_restart(&mut rx, name, EVENT_WAIT).await;
assert_eq!(info.restarts, 1);
group.abort();
}
#[tokio::test(start_paused = true)]
async fn a_batch_queued_during_a_restart_is_rechecked_and_drained_as_one() {
let (handle, mut rx, _dir) = spawn_test_fixture(vec![ProcScript::ignores_signals(); 4]);
let name = "web";
start_app(&handle, name, 1).await;
let root = PathBuf::from("/watched");
let (tx, group_rx) = mpsc::unbounded_channel();
let group = tokio::spawn(run_group(
name.to_string(),
matches_everything(root.clone()),
group_rx,
handle.clone(),
));
tx.send(changed(vec![root.join("a.rs")])).unwrap();
settle().await;
tx.send(changed(vec![root.join("b.rs")])).unwrap();
tx.send(changed(vec![root.join("c.rs")])).unwrap();
let first = expect_restart(&mut rx, name, EVENT_WAIT).await;
assert_eq!(first.restarts, 1);
let second = expect_restart(&mut rx, name, EVENT_WAIT).await;
assert_eq!(second.restarts, 2);
assert_no_restart_within(&mut rx, name, Duration::from_secs(5)).await;
group.abort();
}
#[tokio::test(start_paused = true)]
async fn dropping_the_sender_ends_the_group_task() {
let (handle, _rx, _dir) = spawn_test_fixture(vec![]);
let root = PathBuf::from("/watched");
let (tx, group_rx) = mpsc::unbounded_channel();
let group = tokio::spawn(run_group(
"ghost".to_string(),
matches_everything(root),
group_rx,
handle,
));
drop(tx);
tokio::time::timeout(EVENT_WAIT, group)
.await
.expect("group task did not end after its sender was dropped")
.expect("group task panicked");
}
#[tokio::test(start_paused = true)]
async fn a_triggering_batch_restarts_a_stopped_instance_in_the_same_group() {
let (handle, mut rx, _dir) = spawn_test_fixture(vec![ProcScript::never_exits(); 4]);
let name = "web";
let infos = start_app(&handle, name, 2).await;
let stopped_id = infos[1].id;
handle.stop(ProcessSelector::Id(stopped_id)).await.unwrap();
let root = PathBuf::from("/watched");
let (tx, group_rx) = mpsc::unbounded_channel();
let group = tokio::spawn(run_group(
name.to_string(),
matches_everything(root.clone()),
group_rx,
handle.clone(),
));
tx.send(changed(vec![root.join("src/main.rs")])).unwrap();
let first = expect_restart(&mut rx, name, EVENT_WAIT).await;
let second = expect_restart(&mut rx, name, EVENT_WAIT).await;
let stopped_info = [first, second]
.into_iter()
.find(|info| info.id == stopped_id)
.expect("the previously-stopped instance never restarted");
assert_eq!(stopped_info.status, ProcStatus::Online);
assert_eq!(stopped_info.restarts, 1);
group.abort();
}
#[tokio::test(start_paused = true)]
async fn an_operators_stop_beats_a_watch_triggered_restart_mid_ladder() {
let (handle, mut rx, _dir) = spawn_test_fixture(vec![
ProcScript::ignores_signals(), ProcScript::never_exits(), ProcScript::never_exits(), ProcScript::never_exits(), ]);
let name = "web";
let infos = start_app(&handle, name, 2).await;
let (held, released) = (infos[0].id, infos[1].id);
let root = PathBuf::from("/watched");
let (tx, group_rx) = mpsc::unbounded_channel();
let group = tokio::spawn(run_group(
name.to_string(),
matches_everything(root.clone()),
group_rx,
handle.clone(),
));
tx.send(changed(vec![root.join("src/main.rs")])).unwrap();
let restarted = expect_restart(&mut rx, name, EVENT_WAIT).await;
assert_eq!(
(restarted.id, restarted.restarts),
(released, 1),
"the batch never reached the actor, so the stop below would race \
nothing -- got {restarted:?}"
);
group.abort();
let stopped = handle.stop(ProcessSelector::Id(held)).await.unwrap();
assert_eq!(stopped.len(), 1);
assert_eq!(
(stopped[0].id, stopped[0].status, stopped[0].restarts),
(held, ProcStatus::Stopped, 0),
"an operator's stop was silently converted into the watch-triggered \
restart it raced -- got {stopped:?}"
);
let listed = handle.list().await;
assert_eq!(
(listed[0].id, listed[0].status, listed[0].pid),
(held, ProcStatus::Stopped, None),
"the sheep an operator stopped is running again -- got {listed:?}"
);
assert_eq!(
(listed[1].id, listed[1].status),
(released, ProcStatus::Online),
"the instance the operator did not name must still be up, \
restarted by the batch -- got {listed:?}"
);
}
#[tokio::test(start_paused = true)]
async fn not_found_leaves_the_loop_alive_for_the_next_batch() {
let (handle, mut rx, _dir) = spawn_test_fixture(vec![ProcScript::never_exits(); 2]);
let name = "ghost";
let root = PathBuf::from("/watched");
let (tx, group_rx) = mpsc::unbounded_channel();
let group = tokio::spawn(run_group(
name.to_string(),
matches_everything(root.clone()),
group_rx,
handle.clone(),
));
tx.send(changed(vec![root.join("a.rs")])).unwrap();
assert_no_restart_within(&mut rx, name, Duration::from_millis(200)).await;
assert!(!group.is_finished(), "the loop must not exit on NotFound");
start_app(&handle, name, 1).await;
tx.send(changed(vec![root.join("b.rs")])).unwrap();
let info = expect_restart(&mut rx, name, EVENT_WAIT).await;
assert_eq!(info.restarts, 1);
group.abort();
}
#[tokio::test(start_paused = true)]
async fn the_group_task_ends_when_the_supervisor_engine_has_stopped() {
let (handle, _rx, _dir) = spawn_test_fixture(Vec::new());
let name = "web";
handle.shutdown().await;
assert_eq!(
handle
.restart_automatic(ProcessSelector::Name(name.to_string()))
.await
.unwrap_err(),
SupervisorError::EngineStopped
);
let root = PathBuf::from("/watched");
let (tx, group_rx) = mpsc::unbounded_channel();
let group = tokio::spawn(run_group(
name.to_string(),
matches_everything(root.clone()),
group_rx,
handle,
));
tx.send(changed(vec![root.join("src/main.rs")])).unwrap();
tokio::time::timeout(EVENT_WAIT, group)
.await
.expect("the group task did not end after the engine shut down")
.expect("group task panicked");
drop(tx); }
#[tokio::test]
async fn a_watch_root_that_does_not_exist_names_the_path_it_could_not_watch() {
let (handle, _rx, _dir) = spawn_test_fixture(vec![]);
let parent = tempfile::tempdir().unwrap();
let missing = parent.path().join("no-such-directory");
let err = spawn_watch_group(
"web".to_string(),
missing.clone(),
WatchFilter::new(&[], &[]).unwrap(),
real_time::TEST_DELAY,
handle,
)
.unwrap_err();
let WatchError::Watch { path, reason } = err else {
panic!("a root that does not exist must report `Watch`, got {err:?}");
};
assert_eq!(path, missing, "`Watch` must carry the root it was handed");
assert!(!reason.is_empty(), "`Watch` must carry notify's own reason");
}
const MAX_BATCHES: usize = 8;
const SINGLE_FLIGHT_SCRIPTS: usize = MAX_BATCHES + 4;
#[derive(Debug, Clone, Copy)]
struct Batch {
triggers: bool,
gap: Duration,
}
fn gap_strategy() -> impl proptest::strategy::Strategy<Value = Duration> {
use proptest::strategy::Strategy as _; proptest::prop_oneof![
4 => proptest::strategy::Just(Duration::ZERO),
3 => (1u64..2_000u64).prop_map(Duration::from_millis),
1 => (2_000u64..6_000u64).prop_map(Duration::from_millis),
]
}
fn batch_strategy() -> impl proptest::strategy::Strategy<Value = Batch> {
use proptest::strategy::Strategy as _; (proptest::bool::ANY, gap_strategy()).prop_map(|(triggers, gap)| Batch { triggers, gap })
}
fn expected_restart_instants(batches: &[Batch], restart: Duration) -> Vec<Duration> {
let mut arrivals = Vec::with_capacity(batches.len());
let mut at = Duration::ZERO;
for batch in batches {
at += batch.gap;
arrivals.push((at, batch.triggers));
}
let mut finished = Vec::new();
let mut idle_since = Duration::ZERO;
let mut i = 0;
while i < arrivals.len() {
let woke_at = arrivals[i].0.max(idle_since);
let mut triggers = arrivals[i].1;
i += 1;
while i < arrivals.len() && arrivals[i].0 <= woke_at {
triggers |= arrivals[i].1;
i += 1;
}
idle_since = if triggers {
let done = woke_at + restart;
finished.push(done);
done
} else {
woke_at
};
}
finished
}
proptest::proptest! {
#![proptest_config(crate::testing::proptest_config(64))]
#[test]
fn a_watch_group_never_has_two_restarts_in_flight(
batches in proptest::collection::vec(batch_strategy(), 1..=MAX_BATCHES),
kill_timeout_ms in 200u64..2_000u64,
) {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.start_paused(true)
.build()
.unwrap();
let dir = tempfile::tempdir().unwrap();
runtime.block_on(async move {
let kill_timeout = Duration::from_millis(kill_timeout_ms);
let (events, mut rx) = crate::bus::test_bus(1024);
let runner =
ScriptedRunner::new(vec![ProcScript::ignores_signals(); SINGLE_FLIGHT_SCRIPTS]);
let handle = spawn_supervisor(runner, test_paths(&dir), events);
let name = "web";
let mut app = AppConfig::minimal(name, "./srv");
app.kill_timeout = UpDuration::from_millis(kill_timeout_ms);
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
let root = PathBuf::from("/watched");
let (tx, group_rx) = mpsc::unbounded_channel();
let group = tokio::spawn(run_group(
name.to_string(),
matches_everything(root.clone()),
group_rx,
handle.clone(),
));
let start = tokio::time::Instant::now();
let watched = name.to_string();
let collector = tokio::spawn(async move {
let mut observed = Vec::new();
loop {
match tokio::time::timeout(EVENT_WAIT, rx.recv()).await.map(|received| received.map(|event| event.to_event())) {
Ok(Ok(BusEvent::Process {
event: ProcessEventKind::Restart,
info,
..
})) if info.name == watched => {
observed
.push((tokio::time::Instant::now() - start, info.restarts));
}
Ok(Ok(_)) => continue,
Ok(Err(broadcast::error::RecvError::Lagged(skipped))) => {
return Err(skipped);
}
Ok(Err(broadcast::error::RecvError::Closed)) => break,
Err(_elapsed) => break, }
}
Ok(observed)
});
for (i, batch) in batches.iter().enumerate() {
if batch.gap > Duration::ZERO {
tokio::time::sleep(batch.gap).await;
}
let path = if batch.triggers {
root.join(format!("src/f{i}.rs"))
} else {
root.join(format!(".git/o{i}"))
};
tx.send(changed(vec![path])).unwrap();
}
let observed = match collector.await.expect("collector task panicked") {
Ok(observed) => observed,
Err(skipped) => {
return Err(proptest::test_runner::TestCaseError::fail(format!(
"event stream lagged by {skipped}"
)));
}
};
group.abort();
for pair in observed.windows(2) {
proptest::prop_assert!(
pair[1].0 - pair[0].0 >= kill_timeout,
"two restarts of {} finished {:?} apart, less than the {:?} one takes: \
they overlapped",
name,
pair[1].0 - pair[0].0,
kill_timeout
);
}
let expected = expected_restart_instants(&batches, kill_timeout);
let counted: Vec<u32> = (1..=expected.len() as u32).collect();
proptest::prop_assert_eq!(
observed,
expected.into_iter().zip(counted).collect::<Vec<_>>()
);
Ok::<(), proptest::test_runner::TestCaseError>(())
})?;
}
}
mod slow {
use super::*;
#[tokio::test]
async fn spawn_watch_group_restarts_on_a_real_touch_and_stops_on_abort() {
let (handle, mut rx, _dir) = spawn_test_fixture(vec![ProcScript::never_exits(); 2]);
let name = "web";
start_app(&handle, name, 1).await;
let watch_dir = tempfile::tempdir().unwrap();
let root = watch_dir.path().canonicalize().unwrap();
let filter = WatchFilter::new(&[], &[]).unwrap();
let group = spawn_watch_group(
name.to_string(),
root.clone(),
filter,
real_time::TEST_DELAY,
handle.clone(),
)
.unwrap();
crate::testing::touch(&root, "trigger.txt").unwrap();
let info = expect_restart(&mut rx, name, real_time::SMOKE_DEADLINE).await;
assert_eq!(info.restarts, 1);
group.abort();
crate::testing::touch(&root, "after-abort.txt").unwrap();
assert_no_restart_within(&mut rx, name, real_time::NO_EVENT_WINDOW).await;
}
}
}