use std::collections::{HashMap, VecDeque};
use std::fmt;
use std::sync::Arc;
use std::time::{Duration, Instant};
use acton_ern::Ern;
use tokio::sync::watch;
use super::plan::{ExpectedTermination, SlotSnapshot, SlotView};
use super::{ChildSpawner, SupervisionError, SupervisionState, SupervisionStatus};
use crate::actor::{RestartLimiter, RestartPolicy};
use crate::common::ActorHandle;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
pub struct RestartGeneration(u64);
impl RestartGeneration {
pub const FIRST: Self = Self(0);
#[must_use]
pub const fn next(self) -> Self {
Self(self.0.wrapping_add(1))
}
#[must_use]
pub const fn get(self) -> u64 {
self.0
}
}
impl fmt::Display for RestartGeneration {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "generation {}", self.0)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct ChildIndex(usize);
impl ChildIndex {
#[must_use]
pub const fn new(index: usize) -> Self {
Self(index)
}
#[must_use]
pub const fn get(self) -> usize {
self.0
}
}
impl From<ChildIndex> for usize {
fn from(value: ChildIndex) -> Self {
value.0
}
}
impl fmt::Display for ChildIndex {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "child index {}", self.0)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Default)]
pub struct BackoffDelay(Duration);
impl BackoffDelay {
pub const NONE: Self = Self(Duration::ZERO);
#[must_use]
pub const fn duration(self) -> Duration {
self.0
}
#[must_use]
pub const fn is_immediate(self) -> bool {
self.0.is_zero()
}
}
impl From<Duration> for BackoffDelay {
fn from(value: Duration) -> Self {
Self(value)
}
}
impl From<BackoffDelay> for Duration {
fn from(value: BackoffDelay) -> Self {
value.0
}
}
impl fmt::Display for BackoffDelay {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}ms", self.0.as_millis())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SlotState {
Pending,
Starting,
Running,
AwaitingBackoff,
AwaitingRestart,
Restarting,
ExpectedStop {
then_restart: bool,
},
Down,
Escalated,
Retired,
}
impl SlotState {
#[must_use]
pub const fn published(self) -> SupervisionState {
match self {
Self::Pending | Self::Starting => SupervisionState::Starting,
Self::Running => SupervisionState::Running,
Self::AwaitingBackoff => SupervisionState::RestartPending,
Self::AwaitingRestart | Self::Restarting | Self::ExpectedStop { .. } => {
SupervisionState::Restarting
}
Self::Down => SupervisionState::Down,
Self::Escalated => SupervisionState::Escalated,
Self::Retired => SupervisionState::Retired,
}
}
#[must_use]
pub const fn is_running(self) -> bool {
matches!(self, Self::Starting | Self::Running)
}
#[must_use]
pub const fn accepts_termination(self) -> bool {
matches!(self, Self::Starting | Self::Running)
}
#[must_use]
pub const fn is_being_started(self) -> bool {
matches!(self, Self::Starting | Self::Restarting)
}
#[must_use]
pub const fn is_queued_to_start(self) -> bool {
matches!(self, Self::Pending | Self::AwaitingRestart)
}
#[must_use]
pub const fn is_unfinished_start(self) -> bool {
self.is_queued_to_start() || self.is_being_started()
}
#[must_use]
pub const fn is_awaiting_backoff(self) -> bool {
matches!(self, Self::AwaitingBackoff)
}
#[must_use]
pub const fn is_awaiting_group_restart(self) -> bool {
matches!(self, Self::ExpectedStop { .. })
}
}
impl fmt::Display for SlotState {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Pending => f.write_str("pending"),
Self::Starting => f.write_str("starting"),
Self::Running => f.write_str("running"),
Self::AwaitingBackoff => f.write_str("awaiting_backoff"),
Self::Restarting => f.write_str("restarting"),
Self::ExpectedStop { then_restart } => {
write!(f, "expected_stop(then_restart={then_restart})")
}
Self::AwaitingRestart => f.write_str("awaiting_restart"),
Self::Down => f.write_str("down"),
Self::Escalated => f.write_str("escalated"),
Self::Retired => f.write_str("retired"),
}
}
}
#[derive(Debug)]
pub struct NewSlot {
pub ern: Ern,
pub handle: ActorHandle,
pub spawner: Option<Arc<dyn ChildSpawner>>,
pub restart_policy: RestartPolicy,
pub limiter: RestartLimiter,
pub status: watch::Sender<SupervisionStatus>,
}
#[derive(Debug)]
pub struct PendingSlot {
pub ern: Ern,
pub spawner: Arc<dyn ChildSpawner>,
pub restart_policy: RestartPolicy,
pub limiter: RestartLimiter,
pub status: watch::Sender<SupervisionStatus>,
}
#[derive(Debug, Clone)]
pub struct StartTicket {
pub index: ChildIndex,
pub ern: Ern,
pub spawner: Arc<dyn ChildSpawner>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum StartRecorded {
First,
Restart,
Refused,
}
impl StartRecorded {
#[must_use]
pub const fn is_recorded(self) -> bool {
matches!(self, Self::First | Self::Restart)
}
}
impl fmt::Display for StartRecorded {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::First => f.write_str("first start"),
Self::Restart => f.write_str("restart"),
Self::Refused => f.write_str("refused"),
}
}
}
#[derive(Debug)]
pub struct ChildSlot {
ern: Ern,
index: ChildIndex,
handle: Option<ActorHandle>,
spawner: Option<Arc<dyn ChildSpawner>>,
restart_policy: RestartPolicy,
limiter: RestartLimiter,
generation: RestartGeneration,
state: SlotState,
last_restart: Option<Instant>,
status: watch::Sender<SupervisionStatus>,
failure: Option<SupervisionError>,
}
impl ChildSlot {
pub const fn ern(&self) -> &Ern {
&self.ern
}
pub const fn index(&self) -> ChildIndex {
self.index
}
pub const fn handle(&self) -> Option<&ActorHandle> {
self.handle.as_ref()
}
pub const fn restart_policy(&self) -> RestartPolicy {
self.restart_policy
}
pub const fn generation(&self) -> RestartGeneration {
self.generation
}
pub const fn state(&self) -> SlotState {
self.state
}
pub const fn last_restart(&self) -> Option<Instant> {
self.last_restart
}
pub const fn is_restartable(&self) -> bool {
self.spawner.is_some()
}
pub const fn is_pending(&self) -> bool {
matches!(self.state, SlotState::Pending)
}
pub const fn is_starting(&self) -> bool {
self.state.is_being_started()
}
pub const fn is_awaiting_backoff(&self) -> bool {
self.state.is_awaiting_backoff()
}
pub const fn is_owed_an_incarnation(&self) -> bool {
self.state.is_unfinished_start()
|| self.state.is_awaiting_backoff()
|| self.state.is_awaiting_group_restart()
}
pub const fn failure(&self) -> Option<&SupervisionError> {
self.failure.as_ref()
}
pub fn set_failure(&mut self, failure: SupervisionError) {
self.failure = Some(failure);
}
pub fn spawner(&self) -> Option<Arc<dyn ChildSpawner>> {
self.spawner.clone()
}
pub const fn limiter_mut(&mut self) -> &mut RestartLimiter {
&mut self.limiter
}
pub const fn set_state(&mut self, state: SlotState) {
self.state = state;
}
pub fn set_handle(&mut self, handle: Option<ActorHandle>) {
self.handle = handle;
}
pub const fn advance_generation(&mut self) {
self.generation = self.generation.next();
}
pub const fn mark_restarted_at(&mut self, at: Instant) {
self.last_restart = Some(at);
}
pub fn publish(&self) -> bool {
let mut next = SupervisionStatus::new(
self.ern.clone(),
self.handle.clone(),
self.generation,
self.state.published(),
self.limiter.restarts_in_window(),
);
if let Some(failure) = self.failure.clone() {
next = next.with_failure(failure);
}
self.status.send_if_modified(|current| {
let unchanged = current.generation() == next.generation()
&& current.state() == next.state()
&& current.restarts_in_window() == next.restarts_in_window()
&& current.failure() == next.failure();
if unchanged {
return false;
}
*current = next;
true
})
}
const fn view(&self) -> SlotView {
SlotView {
index: self.index,
restartable: self.is_restartable(),
alive: self.handle.is_some() && self.state.is_running(),
}
}
}
#[derive(Debug, Default)]
pub struct SupervisionRegistry {
slots: Vec<ChildSlot>,
by_ern: HashMap<Ern, ChildIndex>,
pending_starts: VecDeque<ChildIndex>,
shutting_down: bool,
}
impl SupervisionRegistry {
pub fn register(&mut self, new: NewSlot) -> Result<ChildIndex, SupervisionError> {
if self.by_ern.contains_key(&new.ern) {
return Err(SupervisionError::DuplicateChild { child: new.ern });
}
let index = ChildIndex::new(self.slots.len());
let slot = ChildSlot {
ern: new.ern.clone(),
index,
handle: Some(new.handle),
spawner: new.spawner,
restart_policy: new.restart_policy,
limiter: new.limiter,
generation: RestartGeneration::FIRST,
state: SlotState::Running,
last_restart: None,
status: new.status,
failure: None,
};
slot.publish();
self.slots.push(slot);
self.by_ern.insert(new.ern, index);
Ok(index)
}
pub fn register_pending(&mut self, new: PendingSlot) -> Result<ChildIndex, SupervisionError> {
if self.by_ern.contains_key(&new.ern) {
return Err(SupervisionError::DuplicateChild { child: new.ern });
}
let index = ChildIndex::new(self.slots.len());
let slot = ChildSlot {
ern: new.ern.clone(),
index,
handle: None,
spawner: Some(new.spawner),
restart_policy: new.restart_policy,
limiter: new.limiter,
generation: RestartGeneration::FIRST,
state: SlotState::Pending,
last_restart: None,
status: new.status,
failure: None,
};
slot.publish();
self.slots.push(slot);
self.by_ern.insert(new.ern, index);
self.pending_starts.push_back(index);
Ok(index)
}
pub fn has_pending_starts(&self) -> bool {
!self.pending_starts.is_empty()
}
pub fn begin_start(&mut self) -> Option<StartTicket> {
while let Some(index) = self.pending_starts.pop_front() {
let Some(slot) = self.slots.get_mut(index.get()) else {
continue;
};
let launched = match slot.state() {
SlotState::Pending => SlotState::Starting,
SlotState::AwaitingRestart => SlotState::Restarting,
_ => continue,
};
let Some(spawner) = slot.spawner() else {
continue;
};
slot.set_state(launched);
slot.publish();
return Some(StartTicket {
index,
ern: slot.ern.clone(),
spawner,
});
}
None
}
pub fn complete_start(
&mut self,
index: ChildIndex,
ern: &Ern,
handle: ActorHandle,
now: Instant,
) -> StartRecorded {
let Some(slot) = self.slots.get_mut(index.get()) else {
return StartRecorded::Refused;
};
let restarted = match slot.state() {
SlotState::Starting => false,
SlotState::Restarting => true,
_ => return StartRecorded::Refused,
};
if &slot.ern != ern {
return StartRecorded::Refused;
}
slot.set_handle(Some(handle));
if restarted {
slot.advance_generation();
slot.mark_restarted_at(now);
}
slot.set_state(SlotState::Running);
slot.publish();
if restarted {
StartRecorded::Restart
} else {
StartRecorded::First
}
}
pub fn queue_restart(
&mut self,
index: ChildIndex,
ern: &Ern,
generation: RestartGeneration,
) -> bool {
let Some(slot) = self.slots.get_mut(index.get()) else {
return false;
};
if !slot.is_awaiting_backoff() || &slot.ern != ern || slot.generation() != generation {
return false;
}
slot.set_state(SlotState::AwaitingRestart);
slot.publish();
self.pending_starts.push_back(index);
true
}
pub fn mark_terminal(
&mut self,
index: ChildIndex,
state: SlotState,
failure: Option<SupervisionError>,
) {
debug_assert!(
matches!(state, SlotState::Down | SlotState::Escalated),
"mark_terminal records why a child stopped, not that it was released"
);
let Some(slot) = self.slots.get_mut(index.get()) else {
return;
};
slot.set_handle(None);
if let Some(failure) = failure {
slot.set_failure(failure);
}
slot.set_state(state);
slot.publish();
}
pub fn fail_start(&mut self, index: ChildIndex, failure: &SupervisionError) {
let Some(ern) = self
.slots
.get(index.get())
.filter(|slot| slot.is_owed_an_incarnation())
.map(|slot| slot.ern.clone())
else {
return;
};
self.by_ern.remove(&ern);
let Some(slot) = self.slots.get_mut(index.get()) else {
return;
};
slot.set_handle(None);
slot.set_failure(failure.clone());
slot.set_state(SlotState::Retired);
slot.publish();
}
pub fn cancel_unfinished_starts(&mut self, supervisor: &Ern) -> usize {
self.pending_starts.clear();
let unfinished: Vec<ChildIndex> = self
.slots
.iter()
.filter(|slot| slot.is_owed_an_incarnation())
.map(ChildSlot::index)
.collect();
for index in &unfinished {
self.fail_start(
*index,
&SupervisionError::SupervisorStopped {
supervisor: supervisor.clone(),
},
);
}
unfinished.len()
}
pub fn replace_legacy(
&mut self,
ern: &Ern,
handle: ActorHandle,
) -> Result<(), SupervisionError> {
let index = self
.by_ern
.get(ern)
.copied()
.ok_or_else(|| SupervisionError::UnknownChild { child: ern.clone() })?;
let Some(slot) = self.slots.get_mut(index.get()) else {
return Err(SupervisionError::UnknownChild { child: ern.clone() });
};
slot.set_handle(Some(handle));
slot.set_state(SlotState::Running);
slot.publish();
Ok(())
}
pub fn retire(&mut self, ern: &Ern) -> Result<Option<ActorHandle>, SupervisionError> {
let index = self
.by_ern
.remove(ern)
.ok_or_else(|| SupervisionError::UnknownChild { child: ern.clone() })?;
let Some(slot) = self.slots.get_mut(index.get()) else {
return Err(SupervisionError::UnknownChild { child: ern.clone() });
};
let handle = slot.handle.take();
slot.set_state(SlotState::Retired);
slot.publish();
Ok(handle)
}
pub fn index_of(&self, ern: &Ern) -> Option<ChildIndex> {
self.by_ern.get(ern).copied()
}
pub fn slot(&self, index: ChildIndex) -> Option<&ChildSlot> {
self.slots.get(index.get())
}
pub fn slot_mut(&mut self, index: ChildIndex) -> Option<&mut ChildSlot> {
self.slots.get_mut(index.get())
}
pub fn slot_of(&self, ern: &Ern) -> Option<&ChildSlot> {
self.index_of(ern).and_then(|index| self.slot(index))
}
pub fn slot_of_mut(&mut self, ern: &Ern) -> Option<&mut ChildSlot> {
self.index_of(ern).and_then(|index| self.slot_mut(index))
}
pub fn views(&self) -> Vec<SlotView> {
self.slots
.iter()
.filter(|slot| slot.state != SlotState::Retired)
.map(ChildSlot::view)
.collect()
}
pub fn snapshot(&self, index: ChildIndex) -> Option<SlotSnapshot> {
let slot = self.slot(index)?;
let expected = if self.shutting_down {
Some(ExpectedTermination::Shutdown)
} else if let SlotState::ExpectedStop { then_restart } = slot.state {
Some(ExpectedTermination::GroupStop { then_restart })
} else if slot.state.accepts_termination() {
None
} else {
Some(ExpectedTermination::Stale)
};
Some(SlotSnapshot {
index: slot.index,
restartable: slot.is_restartable(),
expected,
last_restart: slot.last_restart,
})
}
pub fn live_handles(&self) -> Vec<ActorHandle> {
self.slots
.iter()
.filter(|slot| slot.state != SlotState::Retired)
.filter_map(|slot| slot.handle.clone())
.collect()
}
pub fn engine_managed_children(&self) -> Vec<Ern> {
self.slots
.iter()
.filter(|slot| slot.state != SlotState::Retired && slot.is_restartable())
.map(|slot| slot.ern.clone())
.collect()
}
pub fn is_empty(&self) -> bool {
self.by_ern.is_empty()
}
pub fn len(&self) -> usize {
self.by_ern.len()
}
pub const fn begin_shutdown(&mut self) {
self.shutting_down = true;
}
pub const fn is_shutting_down(&self) -> bool {
self.shutting_down
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::traits::ActorHandleInterface;
#[test]
fn first_generation_is_zero() {
assert_eq!(RestartGeneration::FIRST.get(), 0);
assert_eq!(RestartGeneration::default(), RestartGeneration::FIRST);
}
#[test]
fn generation_advances_monotonically() {
let first = RestartGeneration::FIRST;
let second = first.next();
let third = second.next();
assert!(first < second);
assert!(second < third);
assert_eq!(third.get(), 2);
}
#[test]
fn generation_wraps_instead_of_panicking_at_the_maximum() {
let highest = RestartGeneration::FIRST.next();
assert_eq!(highest.get(), 1);
let mut at_max = RestartGeneration::FIRST;
for _ in 0..3 {
at_max = at_max.next();
}
assert_eq!(at_max.get(), 3);
}
#[test]
fn generation_displays_with_its_counter() {
assert_eq!(RestartGeneration::FIRST.next().next().next().to_string(), "generation 3");
}
#[test]
fn child_index_round_trips_through_usize() {
let index = ChildIndex::new(7);
assert_eq!(index.get(), 7);
assert_eq!(usize::from(index), 7);
}
#[test]
fn child_index_orders_by_start_position() {
assert!(ChildIndex::new(1) < ChildIndex::new(2));
assert_eq!(ChildIndex::new(2), ChildIndex::new(2));
}
#[test]
fn child_index_displays_with_its_position() {
assert_eq!(ChildIndex::new(2).to_string(), "child index 2");
}
#[test]
fn no_backoff_is_immediate() {
assert!(BackoffDelay::NONE.is_immediate());
assert_eq!(BackoffDelay::NONE.duration(), Duration::ZERO);
assert_eq!(BackoffDelay::default(), BackoffDelay::NONE);
}
#[test]
fn nonzero_backoff_is_not_immediate() {
let delay = BackoffDelay::from(Duration::from_millis(250));
assert!(!delay.is_immediate());
assert_eq!(delay.duration(), Duration::from_millis(250));
assert_eq!(Duration::from(delay), Duration::from_millis(250));
}
#[test]
fn backoff_orders_by_duration() {
assert!(BackoffDelay::from(Duration::from_millis(100))
< BackoffDelay::from(Duration::from_millis(200)));
}
#[test]
fn backoff_displays_in_milliseconds() {
assert_eq!(BackoffDelay::from(Duration::from_millis(250)).to_string(), "250ms");
assert_eq!(BackoffDelay::NONE.to_string(), "0ms");
}
#[derive(Debug)]
struct StubSpawner {
child: Ern,
}
impl ChildSpawner for StubSpawner {
fn child_id(&self) -> &Ern {
&self.child
}
fn restart_policy(&self) -> RestartPolicy {
RestartPolicy::Permanent
}
fn spawn(
&self,
_runtime: crate::common::ActorRuntime,
_parent: ActorHandle,
) -> std::pin::Pin<
Box<
dyn std::future::Future<Output = Result<ActorHandle, SupervisionError>>
+ Send
+ '_,
>,
> {
Box::pin(async move {
Err(SupervisionError::ConfigRejected {
child: self.child.clone(),
reason: "stub spawner never builds an actor".to_string(),
})
})
}
}
fn ern(name: &str) -> Ern {
Ern::with_root(name).expect("valid Ern root")
}
fn handle(id: &Ern) -> ActorHandle {
let (outbox, _inbox) = tokio::sync::mpsc::channel(8);
ActorHandle::new(id.clone(), outbox)
}
fn status_channel(
id: &Ern,
) -> (
watch::Sender<SupervisionStatus>,
watch::Receiver<SupervisionStatus>,
) {
watch::channel(SupervisionStatus::new(
id.clone(),
None,
RestartGeneration::FIRST,
SupervisionState::Starting,
0,
))
}
fn new_slot(id: &Ern, restartable: bool) -> (NewSlot, watch::Receiver<SupervisionStatus>) {
let (status, receiver) = status_channel(id);
let spawner: Option<Arc<dyn ChildSpawner>> = if restartable {
Some(Arc::new(StubSpawner { child: id.clone() }))
} else {
None
};
(
NewSlot {
ern: id.clone(),
handle: handle(id),
spawner,
restart_policy: RestartPolicy::Permanent,
limiter: RestartLimiter::default(),
status,
},
receiver,
)
}
fn registry_with(count: usize) -> (SupervisionRegistry, Vec<Ern>) {
let mut registry = SupervisionRegistry::default();
let mut erns = Vec::new();
for _ in 0..count {
let id = ern("child");
let (slot, _receiver) = new_slot(&id, true);
registry.register(slot).expect("distinct Erns never collide");
erns.push(id);
}
(registry, erns)
}
#[test]
fn a_spawner_reports_the_identity_and_policy_it_will_build_with() {
let id = ern("child");
let spawner = StubSpawner { child: id.clone() };
assert_eq!(spawner.child_id(), &id);
assert_eq!(spawner.restart_policy(), RestartPolicy::Permanent);
}
#[test]
fn every_slot_state_maps_to_a_published_state() {
assert_eq!(SlotState::Starting.published(), SupervisionState::Starting);
assert_eq!(SlotState::Running.published(), SupervisionState::Running);
assert_eq!(
SlotState::AwaitingBackoff.published(),
SupervisionState::RestartPending
);
assert_eq!(
SlotState::Restarting.published(),
SupervisionState::Restarting
);
assert_eq!(SlotState::Down.published(), SupervisionState::Down);
assert_eq!(SlotState::Escalated.published(), SupervisionState::Escalated);
assert_eq!(SlotState::Retired.published(), SupervisionState::Retired);
}
#[test]
fn an_expected_stop_publishes_as_restarting_either_way() {
for then_restart in [true, false] {
assert_eq!(
SlotState::ExpectedStop { then_restart }.published(),
SupervisionState::Restarting
);
}
}
#[test]
fn only_starting_and_running_count_as_up() {
assert!(SlotState::Starting.is_running());
assert!(SlotState::Running.is_running());
for state in [
SlotState::AwaitingBackoff,
SlotState::Restarting,
SlotState::ExpectedStop { then_restart: true },
SlotState::Down,
SlotState::Escalated,
SlotState::Retired,
] {
assert!(!state.is_running(), "{state} should not count as up");
}
}
#[test]
fn a_termination_is_only_fresh_news_while_the_child_was_up() {
assert!(SlotState::Starting.accepts_termination());
assert!(SlotState::Running.accepts_termination());
for state in [
SlotState::AwaitingBackoff,
SlotState::Restarting,
SlotState::ExpectedStop { then_restart: true },
SlotState::ExpectedStop {
then_restart: false,
},
SlotState::Down,
SlotState::Escalated,
SlotState::Retired,
] {
assert!(
!state.accepts_termination(),
"{state} should ignore a termination notice"
);
}
}
#[test]
fn registering_assigns_positions_in_call_order() {
let (registry, erns) = registry_with(3);
for (position, id) in erns.iter().enumerate() {
assert_eq!(registry.index_of(id), Some(ChildIndex::new(position)));
}
assert_eq!(registry.len(), 3);
assert!(!registry.is_empty());
}
#[test]
fn registering_publishes_the_child_as_running() {
let id = ern("child");
let (slot, receiver) = new_slot(&id, true);
let mut registry = SupervisionRegistry::default();
assert_eq!(receiver.borrow().state(), SupervisionState::Starting);
registry.register(slot).expect("first registration succeeds");
let published = receiver.borrow().clone();
assert_eq!(published.state(), SupervisionState::Running);
assert_eq!(published.generation(), RestartGeneration::FIRST);
assert!(published.handle().is_some());
}
#[test]
fn registering_the_same_child_twice_is_rejected() {
let id = ern("child");
let mut registry = SupervisionRegistry::default();
let (first, _first_rx) = new_slot(&id, true);
registry.register(first).expect("first registration succeeds");
let (second, _second_rx) = new_slot(&id, true);
let error = registry
.register(second)
.expect_err("the same Ern cannot be registered twice");
assert_eq!(error, SupervisionError::DuplicateChild { child: id });
assert_eq!(registry.len(), 1, "the rejected slot was not recorded");
}
#[test]
fn an_empty_registry_reports_empty() {
let registry = SupervisionRegistry::default();
assert!(registry.is_empty());
assert_eq!(registry.len(), 0);
assert!(registry.views().is_empty());
assert!(registry.live_handles().is_empty());
assert!(!registry.is_shutting_down());
}
fn pending_slot(id: &Ern) -> (PendingSlot, watch::Receiver<SupervisionStatus>) {
let (status, receiver) = status_channel(id);
(
PendingSlot {
ern: id.clone(),
spawner: Arc::new(StubSpawner { child: id.clone() }),
restart_policy: RestartPolicy::Permanent,
limiter: RestartLimiter::default(),
status,
},
receiver,
)
}
#[test]
fn a_pending_child_is_recorded_without_a_handle_and_queued() {
let id = ern("child");
let (slot, receiver) = pending_slot(&id);
let mut registry = SupervisionRegistry::default();
let index = registry
.register_pending(slot)
.expect("first registration succeeds");
let child = registry.slot(index).expect("the slot exists");
assert_eq!(child.state(), SlotState::Pending);
assert!(child.is_pending());
assert!(child.handle().is_none(), "nothing has been created yet");
assert!(child.is_restartable(), "a pending child always has a spawner");
assert_eq!(
receiver.borrow().state(),
SupervisionState::Starting,
"pending is indistinguishable from starting to a caller"
);
assert!(registry.has_pending_starts());
assert_eq!(registry.len(), 1, "the name is taken from this moment on");
}
#[test]
fn a_pending_duplicate_is_rejected_before_anything_is_built() {
let id = ern("child");
let mut registry = SupervisionRegistry::default();
let (first, _first_rx) = pending_slot(&id);
registry
.register_pending(first)
.expect("first registration succeeds");
let (second, _second_rx) = pending_slot(&id);
let error = registry
.register_pending(second)
.expect_err("the same Ern cannot be registered twice");
assert_eq!(error, SupervisionError::DuplicateChild { child: id });
assert_eq!(registry.len(), 1, "the rejected slot was not recorded");
let ticket = registry.begin_start().expect("the accepted child is queued");
assert_eq!(ticket.index, ChildIndex::new(0));
assert!(!registry.has_pending_starts(), "and nothing extra was queued");
}
#[test]
fn a_pending_child_collides_with_a_running_one_of_the_same_name() {
let id = ern("child");
let mut registry = SupervisionRegistry::default();
let (running, _running_rx) = new_slot(&id, true);
registry.register(running).expect("registration succeeds");
let (pending, _pending_rx) = pending_slot(&id);
assert!(matches!(
registry.register_pending(pending),
Err(SupervisionError::DuplicateChild { .. })
));
}
#[test]
fn starting_a_pending_child_attaches_its_handle_and_publishes_running() {
let id = ern("child");
let (slot, receiver) = pending_slot(&id);
let mut registry = SupervisionRegistry::default();
let index = registry.register_pending(slot).expect("registration succeeds");
let ticket = registry.begin_start().expect("one start was queued");
assert_eq!(ticket.index, index);
assert_eq!(ticket.ern, id);
assert_eq!(
registry.slot(index).map(ChildSlot::state),
Some(SlotState::Starting),
"handing the child to a start task is recorded"
);
assert_eq!(
receiver.borrow().state(),
SupervisionState::Starting,
"and looks no different from outside"
);
assert!(registry.complete_start(index, &id, handle(&id), Instant::now()).is_recorded());
let child = registry.slot(index).expect("the slot exists");
assert_eq!(child.state(), SlotState::Running);
assert!(child.handle().is_some());
assert_eq!(receiver.borrow().state(), SupervisionState::Running);
assert!(receiver.borrow().failure().is_none());
}
#[test]
fn starting_a_slot_that_moved_on_is_refused() {
let id = ern("child");
let (slot, _receiver) = pending_slot(&id);
let mut registry = SupervisionRegistry::default();
let index = registry.register_pending(slot).expect("registration succeeds");
registry.begin_start().expect("the start is in flight");
registry.retire(&id).expect("the child is supervised");
assert!(!registry.complete_start(index, &id, handle(&id), Instant::now()).is_recorded());
assert!(
!registry
.complete_start(ChildIndex::new(9), &id, handle(&id), Instant::now())
.is_recorded(),
"an index that never existed is refused too"
);
let other = ern("other");
let (slot, _receiver) = pending_slot(&other);
let index = registry.register_pending(slot).expect("registration succeeds");
registry.begin_start().expect("the start is in flight");
assert!(!registry.complete_start(index, &id, handle(&id), Instant::now()).is_recorded());
assert!(registry
.complete_start(index, &other, handle(&other), Instant::now())
.is_recorded());
}
#[test]
fn a_failed_start_retires_the_slot_and_publishes_the_reason() {
let id = ern("child");
let (slot, receiver) = pending_slot(&id);
let mut registry = SupervisionRegistry::default();
let index = registry.register_pending(slot).expect("registration succeeds");
let failure = SupervisionError::ConfigRejected {
child: id.clone(),
reason: "the spawner said no".to_string(),
};
registry.fail_start(index, &failure);
let published = receiver.borrow().clone();
assert!(
published.state().is_terminal(),
"a caller waiting to see it run must stop waiting"
);
assert_eq!(published.state(), SupervisionState::Retired);
assert_eq!(published.failure(), Some(&failure));
assert!(published.handle().is_none());
assert_eq!(
registry.slot(index).and_then(ChildSlot::failure),
Some(&failure),
"the slot keeps the reason it retired"
);
assert_eq!(registry.index_of(&id), None, "the name is free again");
assert_eq!(registry.len(), 0);
}
#[test]
fn cancelling_queued_starts_tells_every_waiting_caller() {
let supervisor = ern("pool");
let mut registry = SupervisionRegistry::default();
let mut receivers = Vec::new();
for _ in 0..3 {
let id = ern("child");
let (slot, receiver) = pending_slot(&id);
registry.register_pending(slot).expect("registration succeeds");
receivers.push(receiver);
}
let abandoned = registry.cancel_unfinished_starts(&supervisor);
assert_eq!(abandoned, 3);
assert!(!registry.has_pending_starts(), "nothing is left holding a blueprint");
assert!(registry.is_empty());
for receiver in &receivers {
let published = receiver.borrow().clone();
assert!(published.state().is_terminal());
assert_eq!(
published.failure(),
Some(&SupervisionError::SupervisorStopped {
supervisor: supervisor.clone()
})
);
}
}
#[test]
fn cancelling_leaves_children_that_already_started_alone() {
let supervisor = ern("pool");
let id = ern("child");
let (slot, receiver) = pending_slot(&id);
let mut registry = SupervisionRegistry::default();
let index = registry.register_pending(slot).expect("registration succeeds");
let ticket = registry.begin_start().expect("one start was queued");
assert!(registry
.complete_start(ticket.index, &id, handle(&id), Instant::now())
.is_recorded());
assert_eq!(registry.cancel_unfinished_starts(&supervisor), 0);
assert_eq!(
registry.slot(index).map(ChildSlot::state),
Some(SlotState::Running)
);
assert!(receiver.borrow().failure().is_none());
assert_eq!(registry.len(), 1);
}
#[test]
fn cancelling_settles_a_child_that_was_waiting_out_a_backoff() {
let supervisor = ern("pool");
let id = ern("child");
let (slot, receiver) = pending_slot(&id);
let mut registry = SupervisionRegistry::default();
let index = registry.register_pending(slot).expect("registration succeeds");
let ticket = registry.begin_start().expect("one start was queued");
assert!(registry
.complete_start(ticket.index, &id, handle(&id), Instant::now())
.is_recorded());
let child = registry.slot_mut(index).expect("the slot exists");
child.set_handle(None);
child.set_state(SlotState::AwaitingBackoff);
child.publish();
assert_eq!(
receiver.borrow().state(),
SupervisionState::RestartPending,
"the caller is watching a restart that is about to become impossible"
);
assert_eq!(
registry.cancel_unfinished_starts(&supervisor),
1,
"a child waiting out a backoff is abandoned like any other start"
);
let published = receiver.borrow().clone();
assert!(
published.state().is_terminal(),
"the caller must stop waiting, not sit on RestartPending"
);
assert_eq!(
published.failure(),
Some(&SupervisionError::SupervisorStopped {
supervisor: supervisor.clone()
}),
"and learn why the restart is not coming"
);
assert!(registry.is_empty());
}
#[test]
fn only_children_with_a_blueprint_count_as_engine_managed() {
let mut registry = SupervisionRegistry::default();
let engine = ern("engine");
let (slot, _rx) = new_slot(&engine, true);
registry.register(slot).expect("registration succeeds");
let legacy = ern("legacy");
let (slot, _rx) = new_slot(&legacy, false);
registry.register(slot).expect("registration succeeds");
assert_eq!(registry.engine_managed_children(), vec![engine.clone()]);
registry.retire(&engine).expect("the child is supervised");
assert!(registry.engine_managed_children().is_empty());
}
#[test]
fn a_pending_child_contributes_no_handle_to_shutdown() {
let (mut registry, erns) = registry_with(2);
let id = ern("not-yet");
let (slot, _receiver) = pending_slot(&id);
registry.register_pending(slot).expect("registration succeeds");
let handles = registry.live_handles();
assert_eq!(handles.len(), 2);
assert!(handles.iter().all(|handle| handle.id() != id));
assert!(erns.iter().all(|ern| handles.iter().any(|h| &h.id() == ern)));
}
#[test]
fn a_pending_child_is_neither_up_nor_a_source_of_terminations() {
assert!(!SlotState::Pending.is_running());
assert!(!SlotState::Pending.accepts_termination());
assert_eq!(SlotState::Pending.published(), SupervisionState::Starting);
assert_eq!(SlotState::Pending.to_string(), "pending");
}
#[test]
fn retiring_returns_the_handle_so_the_caller_can_stop_the_child() {
let (mut registry, erns) = registry_with(1);
let handle = registry
.retire(&erns[0])
.expect("the child is supervised")
.expect("the child was running");
assert_eq!(handle.id(), erns[0]);
}
#[test]
fn retiring_leaves_later_children_at_their_original_positions() {
let (mut registry, erns) = registry_with(4);
registry.retire(&erns[1]).expect("the child is supervised");
assert_eq!(registry.index_of(&erns[0]), Some(ChildIndex::new(0)));
assert_eq!(registry.index_of(&erns[2]), Some(ChildIndex::new(2)));
assert_eq!(registry.index_of(&erns[3]), Some(ChildIndex::new(3)));
}
#[test]
fn a_retired_child_is_no_longer_supervised() {
let (mut registry, erns) = registry_with(2);
registry.retire(&erns[0]).expect("the child is supervised");
assert_eq!(registry.index_of(&erns[0]), None);
assert_eq!(registry.len(), 1);
assert_eq!(
registry.slot(ChildIndex::new(0)).map(ChildSlot::state),
Some(SlotState::Retired),
"the slot is still there, holding its position"
);
}
#[test]
fn retiring_frees_the_identifier_for_reuse() {
let id = ern("child");
let mut registry = SupervisionRegistry::default();
let (first, _first_rx) = new_slot(&id, true);
registry.register(first).expect("first registration succeeds");
registry.retire(&id).expect("the child is supervised");
let (second, _second_rx) = new_slot(&id, true);
let index = registry
.register(second)
.expect("the identifier was released by retiring");
assert_eq!(
index,
ChildIndex::new(1),
"re-registration takes a fresh position, behind the retired slot"
);
}
#[test]
fn retiring_a_child_that_is_already_down_yields_no_handle() {
let (mut registry, erns) = registry_with(1);
registry
.slot_of_mut(&erns[0])
.expect("the child is supervised")
.set_handle(None);
let handle = registry.retire(&erns[0]).expect("the child is supervised");
assert!(handle.is_none());
}
#[test]
fn retiring_an_unknown_child_is_an_error() {
let mut registry = SupervisionRegistry::default();
let missing = ern("nobody");
let error = registry
.retire(&missing)
.expect_err("nothing was ever registered");
assert_eq!(
error,
SupervisionError::UnknownChild {
child: missing.clone()
}
);
let (mut registry, erns) = registry_with(1);
registry.retire(&erns[0]).expect("the child is supervised");
assert!(matches!(
registry.retire(&erns[0]),
Err(SupervisionError::UnknownChild { .. })
));
}
#[test]
fn replacing_a_legacy_handle_points_the_slot_at_the_new_mailbox() {
let (mut registry, erns) = registry_with(1);
let replacement = handle(&erns[0]);
registry
.replace_legacy(&erns[0], replacement)
.expect("the child is supervised");
let slot = registry.slot_of(&erns[0]).expect("the child is supervised");
assert_eq!(slot.state(), SlotState::Running);
assert!(slot.handle().is_some());
}
#[test]
fn replacing_the_handle_of_an_unknown_child_is_an_error() {
let mut registry = SupervisionRegistry::default();
let missing = ern("nobody");
let error = registry
.replace_legacy(&missing, handle(&missing))
.expect_err("nothing was ever registered");
assert_eq!(error, SupervisionError::UnknownChild { child: missing });
}
#[test]
fn views_describe_every_actively_supervised_child() {
let (registry, _erns) = registry_with(3);
let views = registry.views();
assert_eq!(views.len(), 3);
for (position, view) in views.iter().enumerate() {
assert_eq!(view.index, ChildIndex::new(position));
assert!(view.restartable, "every fixture child has a spawner");
assert!(view.alive, "every fixture child is running");
}
}
#[test]
fn a_child_without_a_spawner_is_not_restartable() {
let id = ern("legacy");
let (slot, _receiver) = new_slot(&id, false);
let mut registry = SupervisionRegistry::default();
registry.register(slot).expect("registration succeeds");
let views = registry.views();
assert_eq!(views.len(), 1);
assert!(!views[0].restartable);
assert!(views[0].alive, "it is running, it just cannot be recreated");
}
#[test]
fn views_omit_retired_slots_but_keep_everyone_elses_position() {
let (mut registry, erns) = registry_with(4);
registry.retire(&erns[1]).expect("the child is supervised");
let views = registry.views();
let positions: Vec<usize> = views.iter().map(|view| view.index.get()).collect();
assert_eq!(
positions,
vec![0, 2, 3],
"the retired slot is gone from planning, the rest keep their indices"
);
}
#[test]
fn a_child_that_is_down_is_not_alive() {
let (mut registry, erns) = registry_with(2);
let slot = registry
.slot_of_mut(&erns[0])
.expect("the child is supervised");
slot.set_state(SlotState::Down);
slot.set_handle(None);
let views = registry.views();
assert!(!views[0].alive);
assert!(views[1].alive);
}
#[test]
fn a_snapshot_of_a_running_child_reads_as_a_fresh_failure() {
let (registry, erns) = registry_with(1);
let index = registry.index_of(&erns[0]).expect("supervised");
let snapshot = registry.snapshot(index).expect("the slot exists");
assert_eq!(snapshot.index, index);
assert!(snapshot.restartable);
assert_eq!(
snapshot.expected, None,
"a running child terminating is news"
);
assert!(snapshot.last_restart.is_none());
}
#[test]
fn a_snapshot_of_a_child_we_stopped_for_a_group_says_so_and_says_which_half() {
for then_restart in [true, false] {
let (mut registry, erns) = registry_with(1);
registry
.slot_of_mut(&erns[0])
.expect("supervised")
.set_state(SlotState::ExpectedStop { then_restart });
let index = registry.index_of(&erns[0]).expect("supervised");
let snapshot = registry.snapshot(index).expect("the slot exists");
assert_eq!(
snapshot.expected,
Some(ExpectedTermination::GroupStop { then_restart })
);
}
}
#[test]
fn a_snapshot_of_a_slot_that_could_not_have_produced_the_notice_reads_as_stale() {
let (mut registry, erns) = registry_with(1);
registry
.slot_of_mut(&erns[0])
.expect("supervised")
.set_state(SlotState::Down);
let index = registry.index_of(&erns[0]).expect("supervised");
let snapshot = registry.snapshot(index).expect("the slot exists");
assert_eq!(snapshot.expected, Some(ExpectedTermination::Stale));
}
#[test]
fn shutting_down_makes_every_termination_expected() {
let (mut registry, erns) = registry_with(2);
registry.begin_shutdown();
assert!(registry.is_shutting_down());
for id in &erns {
let index = registry.index_of(id).expect("supervised");
let snapshot = registry.snapshot(index).expect("the slot exists");
assert_eq!(snapshot.expected, Some(ExpectedTermination::Shutdown));
}
}
#[test]
fn a_shutdown_outranks_a_group_stop_in_the_reason_it_reports() {
let (mut registry, erns) = registry_with(1);
registry
.slot_of_mut(&erns[0])
.expect("supervised")
.set_state(SlotState::ExpectedStop { then_restart: true });
registry.begin_shutdown();
let index = registry.index_of(&erns[0]).expect("supervised");
let snapshot = registry.snapshot(index).expect("the slot exists");
assert_eq!(
snapshot.expected,
Some(ExpectedTermination::Shutdown),
"a shutdown must outrank the group restart it interrupted"
);
}
#[test]
fn a_shutdown_settles_a_child_left_part_way_through_a_group_restart() {
let (mut registry, erns) = registry_with(2);
registry
.slot_of_mut(&erns[0])
.expect("supervised")
.set_state(SlotState::ExpectedStop { then_restart: true });
registry
.slot_of_mut(&erns[1])
.expect("supervised")
.set_state(SlotState::ExpectedStop {
then_restart: false,
});
let abandoned = registry.cancel_unfinished_starts(&ern("supervisor"));
assert_eq!(
abandoned, 2,
"both halves of the group publish as Restarting, so both have a caller waiting"
);
for index in 0..2 {
assert_eq!(
registry
.slot(ChildIndex::new(index))
.expect("the slot exists")
.state(),
SlotState::Retired,
"the caller waiting on slot {index} is told the supervisor stopped"
);
}
}
#[test]
fn a_snapshot_of_a_missing_slot_is_none() {
let (registry, _erns) = registry_with(1);
assert!(registry.snapshot(ChildIndex::new(9)).is_none());
}
#[test]
fn an_out_of_range_index_yields_no_slot_rather_than_panicking() {
let (mut registry, _erns) = registry_with(1);
assert!(registry.slot(ChildIndex::new(9)).is_none());
assert!(registry.slot_mut(ChildIndex::new(9)).is_none());
}
#[test]
fn live_handles_covers_every_child_still_holding_a_mailbox() {
let (registry, _erns) = registry_with(3);
assert_eq!(registry.live_handles().len(), 3);
}
#[test]
fn live_handles_skips_children_that_are_down_or_retired() {
let (mut registry, erns) = registry_with(3);
registry
.slot_of_mut(&erns[0])
.expect("supervised")
.set_handle(None);
registry.retire(&erns[1]).expect("supervised");
let handles = registry.live_handles();
assert_eq!(handles.len(), 1);
assert_eq!(handles[0].id(), erns[2]);
}
#[test]
fn republishing_an_unchanged_state_does_not_wake_watchers() {
let id = ern("child");
let (slot, mut receiver) = new_slot(&id, true);
let mut registry = SupervisionRegistry::default();
registry.register(slot).expect("registration succeeds");
assert!(receiver.has_changed().expect("sender is alive"));
let _ = receiver.borrow_and_update();
let woke = registry
.slot_of(&id)
.expect("supervised")
.publish();
assert!(!woke);
assert!(
!receiver.has_changed().expect("sender is alive"),
"an unchanged republish must not wake a watcher"
);
}
#[test]
fn swapping_the_handle_alone_does_not_wake_watchers() {
let id = ern("child");
let (slot, mut receiver) = new_slot(&id, true);
let mut registry = SupervisionRegistry::default();
registry.register(slot).expect("registration succeeds");
let _ = receiver.borrow_and_update();
let replacement = handle(&id);
let child = registry.slot_of_mut(&id).expect("supervised");
child.set_handle(Some(replacement));
let woke = child.publish();
assert!(!woke, "same generation and state, so nothing to report");
}
#[test]
fn advancing_the_generation_wakes_watchers() {
let id = ern("child");
let (slot, mut receiver) = new_slot(&id, true);
let mut registry = SupervisionRegistry::default();
registry.register(slot).expect("registration succeeds");
let _ = receiver.borrow_and_update();
let child = registry.slot_of_mut(&id).expect("supervised");
child.advance_generation();
let woke = child.publish();
assert!(woke);
assert!(receiver.has_changed().expect("sender is alive"));
assert_eq!(
receiver.borrow().generation(),
RestartGeneration::FIRST.next()
);
}
#[test]
fn changing_state_wakes_watchers() {
let id = ern("child");
let (slot, mut receiver) = new_slot(&id, true);
let mut registry = SupervisionRegistry::default();
registry.register(slot).expect("registration succeeds");
let _ = receiver.borrow_and_update();
let child = registry.slot_of_mut(&id).expect("supervised");
child.set_state(SlotState::AwaitingBackoff);
let woke = child.publish();
assert!(woke);
assert_eq!(receiver.borrow().state(), SupervisionState::RestartPending);
}
#[test]
fn retiring_publishes_the_terminal_state() {
let id = ern("child");
let (slot, receiver) = new_slot(&id, true);
let mut registry = SupervisionRegistry::default();
registry.register(slot).expect("registration succeeds");
registry.retire(&id).expect("supervised");
let published = receiver.borrow().clone();
assert_eq!(published.state(), SupervisionState::Retired);
assert!(published.state().is_terminal());
assert!(published.handle().is_none(), "the handle was handed back");
}
#[test]
fn a_slot_reports_what_it_was_registered_with() {
let id = ern("child");
let (slot, _receiver) = new_slot(&id, true);
let mut registry = SupervisionRegistry::default();
let index = registry.register(slot).expect("registration succeeds");
let child = registry.slot(index).expect("the slot exists");
assert_eq!(child.ern(), &id);
assert_eq!(child.index(), index);
assert_eq!(child.restart_policy(), RestartPolicy::Permanent);
assert_eq!(child.generation(), RestartGeneration::FIRST);
assert_eq!(child.state(), SlotState::Running);
assert!(child.is_restartable());
assert!(child.spawner().is_some());
assert!(child.last_restart().is_none());
}
#[test]
fn a_slot_records_when_its_child_last_came_back() {
let (mut registry, erns) = registry_with(1);
let now = Instant::now();
let child = registry.slot_of_mut(&erns[0]).expect("supervised");
child.mark_restarted_at(now);
assert_eq!(child.last_restart(), Some(now));
let index = registry.index_of(&erns[0]).expect("supervised");
assert_eq!(
registry.snapshot(index).expect("the slot exists").last_restart,
Some(now)
);
}
#[test]
fn a_slots_limiter_is_its_own() {
let (mut registry, erns) = registry_with(2);
let first = registry.slot_of_mut(&erns[0]).expect("supervised");
let _ = first.limiter_mut().record_restart();
let _ = first.limiter_mut().record_restart();
assert!(
registry.slot_of(&erns[0]).expect("supervised").publish(),
"the restart count changed, so watchers are told"
);
let second_count = {
let second = registry.slot_of_mut(&erns[1]).expect("supervised");
second.limiter_mut().restarts_in_window()
};
assert_eq!(second_count, 0);
}
}