use std::collections::BTreeMap;
use std::hash::Hasher;
use std::sync::Arc;
use std::time::Duration;
use crate::runtime::{RegionCreateError, RuntimeState, SpawnError};
use crate::types::{Budget, CancelReason, Outcome, RegionId, TaskId, Time};
#[derive(Clone, Eq, Ord, PartialOrd)]
pub struct ChildName(Arc<str>);
impl ChildName {
pub fn new(name: impl Into<Arc<str>>) -> Self {
Self(name.into())
}
#[must_use]
pub fn as_str(&self) -> &str {
&self.0
}
#[must_use]
pub fn strong_count(&self) -> usize {
Arc::strong_count(&self.0)
}
}
impl std::ops::Deref for ChildName {
type Target = str;
fn deref(&self) -> &str {
&self.0
}
}
impl AsRef<str> for ChildName {
fn as_ref(&self) -> &str {
&self.0
}
}
impl std::borrow::Borrow<str> for ChildName {
fn borrow(&self) -> &str {
&self.0
}
}
impl std::hash::Hash for ChildName {
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
(*self.0).hash(state);
}
}
impl PartialEq for ChildName {
fn eq(&self, other: &Self) -> bool {
*self.0 == *other.0
}
}
impl PartialEq<str> for ChildName {
fn eq(&self, other: &str) -> bool {
&*self.0 == other
}
}
impl PartialEq<&str> for ChildName {
fn eq(&self, other: &&str) -> bool {
&*self.0 == *other
}
}
impl PartialEq<String> for ChildName {
fn eq(&self, other: &String) -> bool {
&*self.0 == other.as_str()
}
}
impl PartialEq<ChildName> for str {
fn eq(&self, other: &ChildName) -> bool {
self == &*other.0
}
}
impl PartialEq<ChildName> for &str {
fn eq(&self, other: &ChildName) -> bool {
*self == &*other.0
}
}
impl PartialEq<ChildName> for String {
fn eq(&self, other: &ChildName) -> bool {
self.as_str() == &*other.0
}
}
impl From<&str> for ChildName {
fn from(s: &str) -> Self {
Self(Arc::from(s))
}
}
impl From<String> for ChildName {
fn from(s: String) -> Self {
Self(Arc::from(s))
}
}
impl From<Arc<str>> for ChildName {
fn from(s: Arc<str>) -> Self {
Self(s)
}
}
impl std::fmt::Debug for ChildName {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{:?}", &*self.0)
}
}
impl std::fmt::Display for ChildName {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.0)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub enum SupervisionStrategy {
#[default]
Stop,
Restart(RestartConfig),
Escalate,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RestartConfig {
pub max_restarts: u32,
pub window: Duration,
pub backoff: BackoffStrategy,
pub restart_cost: u64,
pub min_remaining_for_restart: Option<Duration>,
pub min_polls_for_restart: u32,
}
impl Default for RestartConfig {
fn default() -> Self {
Self {
max_restarts: 3,
window: Duration::from_mins(1),
backoff: BackoffStrategy::default(),
restart_cost: 0,
min_remaining_for_restart: None,
min_polls_for_restart: 0,
}
}
}
impl RestartConfig {
#[must_use]
pub fn new(max_restarts: u32, window: Duration) -> Self {
Self {
max_restarts,
window,
backoff: BackoffStrategy::default(),
restart_cost: 0,
min_remaining_for_restart: None,
min_polls_for_restart: 0,
}
}
#[must_use]
pub fn with_backoff(mut self, backoff: BackoffStrategy) -> Self {
self.backoff = backoff;
self
}
#[must_use]
pub fn with_restart_cost(mut self, cost: u64) -> Self {
self.restart_cost = cost;
self
}
#[must_use]
pub fn with_min_remaining(mut self, min: Duration) -> Self {
self.min_remaining_for_restart = Some(min);
self
}
#[must_use]
pub fn with_min_polls(mut self, min_polls: u32) -> Self {
self.min_polls_for_restart = min_polls;
self
}
}
#[derive(Debug, Clone)]
pub enum BackoffStrategy {
None,
Fixed(Duration),
Exponential {
initial: Duration,
max: Duration,
multiplier: f64,
},
}
impl PartialEq for BackoffStrategy {
fn eq(&self, other: &Self) -> bool {
match (self, other) {
(Self::None, Self::None) => true,
(Self::Fixed(a), Self::Fixed(b)) => a == b,
(
Self::Exponential {
initial: i1,
max: m1,
multiplier: mul1,
},
Self::Exponential {
initial: i2,
max: m2,
multiplier: mul2,
},
) => i1 == i2 && m1 == m2 && mul1.to_bits() == mul2.to_bits(),
_ => false,
}
}
}
impl Default for BackoffStrategy {
fn default() -> Self {
Self::Exponential {
initial: Duration::from_millis(100),
max: Duration::from_secs(10),
multiplier: 2.0,
}
}
}
impl Eq for BackoffStrategy {}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Hash)]
pub enum RestartPolicy {
#[default]
OneForOne,
OneForAll,
RestForOne,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum EscalationPolicy {
#[default]
Stop,
Escalate,
ResetCounter,
}
#[derive(Debug, Clone, PartialEq)]
pub struct SupervisionConfig {
pub restart_policy: RestartPolicy,
pub max_restarts: u32,
pub restart_window: Duration,
pub backoff: BackoffStrategy,
pub escalation: EscalationPolicy,
pub storm_threshold: Option<f64>,
}
impl Default for SupervisionConfig {
fn default() -> Self {
Self {
restart_policy: RestartPolicy::OneForOne,
max_restarts: 3,
restart_window: Duration::from_mins(1),
backoff: BackoffStrategy::default(),
escalation: EscalationPolicy::Stop,
storm_threshold: None,
}
}
}
impl SupervisionConfig {
#[must_use]
pub fn new(max_restarts: u32, restart_window: Duration) -> Self {
Self {
restart_policy: RestartPolicy::OneForOne,
max_restarts,
restart_window,
backoff: BackoffStrategy::default(),
escalation: EscalationPolicy::Stop,
storm_threshold: None,
}
}
#[must_use]
pub fn with_storm_threshold(mut self, threshold: f64) -> Self {
validate_storm_threshold(threshold);
self.storm_threshold = Some(threshold);
self
}
#[must_use]
pub fn with_restart_policy(mut self, policy: RestartPolicy) -> Self {
self.restart_policy = policy;
self
}
#[must_use]
pub fn with_backoff(mut self, backoff: BackoffStrategy) -> Self {
self.backoff = backoff;
self
}
#[must_use]
pub fn with_escalation(mut self, escalation: EscalationPolicy) -> Self {
self.escalation = escalation;
self
}
#[must_use]
pub fn one_for_all(max_restarts: u32, restart_window: Duration) -> Self {
Self::new(max_restarts, restart_window).with_restart_policy(RestartPolicy::OneForAll)
}
#[must_use]
pub fn rest_for_one(max_restarts: u32, restart_window: Duration) -> Self {
Self::new(max_restarts, restart_window).with_restart_policy(RestartPolicy::RestForOne)
}
#[must_use]
pub fn restart_tracker(&self) -> RestartTracker {
let restart = RestartConfig::new(self.max_restarts, self.restart_window)
.with_backoff(self.backoff.clone());
let mut tracker_config = RestartTrackerConfig::from_restart(restart);
if let Some(threshold) = self.storm_threshold {
tracker_config = tracker_config.with_storm_detection(threshold);
}
RestartTracker::new(tracker_config)
}
}
impl Eq for SupervisionConfig {}
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub enum NameRegistrationPolicy {
#[default]
None,
Register {
name: String,
collision: NameCollisionPolicy,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum NameCollisionPolicy {
#[default]
Fail,
Replace,
Wait,
}
pub trait ChildStart: Send {
fn start(
&mut self,
scope: &crate::cx::Scope<'static, crate::types::policy::FailFast>,
state: &mut RuntimeState,
cx: &crate::cx::Cx,
) -> Result<TaskId, SpawnError>;
}
impl<F> ChildStart for F
where
F: FnMut(
&crate::cx::Scope<'static, crate::types::policy::FailFast>,
&mut RuntimeState,
&crate::cx::Cx,
) -> Result<TaskId, SpawnError>
+ Send,
{
fn start(
&mut self,
scope: &crate::cx::Scope<'static, crate::types::policy::FailFast>,
state: &mut RuntimeState,
cx: &crate::cx::Cx,
) -> Result<TaskId, SpawnError> {
(self)(scope, state, cx)
}
}
pub struct ChildSpec {
pub name: ChildName,
pub start: Box<dyn ChildStart>,
pub restart: SupervisionStrategy,
pub shutdown_budget: Budget,
pub depends_on: Vec<ChildName>,
pub registration: NameRegistrationPolicy,
pub start_immediately: bool,
pub required: bool,
}
impl std::fmt::Debug for ChildSpec {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ChildSpec")
.field("name", &self.name)
.field("restart", &self.restart)
.field("shutdown_budget", &self.shutdown_budget)
.field("depends_on", &self.depends_on)
.field("registration", &self.registration)
.field("start_immediately", &self.start_immediately)
.field("required", &self.required)
.finish_non_exhaustive()
}
}
impl ChildSpec {
pub fn new<F>(name: impl Into<ChildName>, start: F) -> Self
where
F: ChildStart + 'static,
{
Self {
name: name.into(),
start: Box::new(start),
restart: SupervisionStrategy::default(),
shutdown_budget: Budget::INFINITE,
depends_on: Vec::new(),
registration: NameRegistrationPolicy::None,
start_immediately: true,
required: true,
}
}
#[must_use]
pub fn with_restart(mut self, restart: SupervisionStrategy) -> Self {
self.restart = restart;
self
}
#[must_use]
pub fn with_shutdown_budget(mut self, budget: Budget) -> Self {
self.shutdown_budget = budget;
self
}
#[must_use]
pub fn depends_on(mut self, name: impl Into<ChildName>) -> Self {
self.depends_on.push(name.into());
self
}
#[must_use]
pub fn with_registration(mut self, policy: NameRegistrationPolicy) -> Self {
self.registration = policy;
self
}
#[must_use]
pub fn with_start_immediately(mut self, start: bool) -> Self {
self.start_immediately = start;
self
}
#[must_use]
pub fn with_required(mut self, required: bool) -> Self {
self.required = required;
self
}
#[must_use]
pub fn spec_eq(&self, other: &Self) -> bool {
self.name == other.name
&& self.restart == other.restart
&& self.shutdown_budget == other.shutdown_budget
&& self.depends_on == other.depends_on
&& self.registration == other.registration
&& self.start_immediately == other.start_immediately
&& self.required == other.required
}
#[must_use]
pub fn spec_fingerprint(&self) -> u64 {
let mut hasher = crate::util::DetHasher::default();
hash_child_spec_fields(self, &mut hasher);
std::hash::Hasher::finish(&hasher)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum StartTieBreak {
#[default]
InsertionOrder,
NameLex,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum SupervisorCompileError {
DuplicateChildName(ChildName),
UnknownDependency {
child: ChildName,
depends_on: ChildName,
},
DeferredDependency {
child: ChildName,
depends_on: ChildName,
},
CycleDetected {
remaining: Vec<ChildName>,
},
}
impl std::fmt::Display for SupervisorCompileError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::DuplicateChildName(name) => write!(f, "duplicate child name: {name}"),
Self::UnknownDependency { child, depends_on } => {
write!(f, "child {child} depends on unknown child {depends_on}")
}
Self::DeferredDependency { child, depends_on } => {
write!(
f,
"child {child} is start_immediately but depends on deferred child {depends_on}"
)
}
Self::CycleDetected { remaining } => {
write!(f, "dependency cycle detected among children: ")?;
for (i, name) in remaining.iter().enumerate() {
if i > 0 {
write!(f, ", ")?;
}
write!(f, "{name}")?;
}
Ok(())
}
}
}
}
impl std::error::Error for SupervisorCompileError {}
#[derive(Debug)]
pub enum SupervisorSpawnError {
RegionCreate(RegionCreateError),
ChildStartFailed {
child: ChildName,
err: SpawnError,
region: RegionId,
},
DependencyUnavailable {
child: ChildName,
dependency: ChildName,
dependency_error: Option<SpawnError>,
region: RegionId,
},
}
impl std::fmt::Display for SupervisorSpawnError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::RegionCreate(e) => write!(f, "supervisor region create failed: {e}"),
Self::ChildStartFailed {
child, err, region, ..
} => {
write!(
f,
"child start failed: child={child} region={region:?} err={err}"
)
}
Self::DependencyUnavailable {
child,
dependency,
dependency_error,
region,
} => match dependency_error {
Some(err) => write!(
f,
"child start blocked: child={child} dependency={dependency} region={region:?} cause={err}"
),
None => write!(
f,
"child start blocked: child={child} dependency={dependency} region={region:?}"
),
},
}
}
}
impl std::error::Error for SupervisorSpawnError {}
impl From<RegionCreateError> for SupervisorSpawnError {
fn from(value: RegionCreateError) -> Self {
Self::RegionCreate(value)
}
}
#[derive(Debug)]
pub struct SupervisorBuilder {
name: ChildName,
budget: Option<Budget>,
tie_break: StartTieBreak,
restart_policy: RestartPolicy,
children: Vec<ChildSpec>,
}
impl SupervisorBuilder {
#[must_use]
pub fn new(name: impl Into<ChildName>) -> Self {
Self {
name: name.into(),
budget: None,
tie_break: StartTieBreak::InsertionOrder,
restart_policy: RestartPolicy::OneForOne,
children: Vec::new(),
}
}
#[must_use]
pub fn with_budget(mut self, budget: Budget) -> Self {
self.budget = Some(budget);
self
}
#[must_use]
pub fn with_tie_break(mut self, tie_break: StartTieBreak) -> Self {
self.tie_break = tie_break;
self
}
#[must_use]
pub fn with_restart_policy(mut self, restart_policy: RestartPolicy) -> Self {
self.restart_policy = restart_policy;
self
}
#[must_use]
pub fn child(mut self, child: ChildSpec) -> Self {
self.children.push(child);
self
}
#[must_use]
pub fn spec_eq(&self, other: &Self) -> bool {
self.name == other.name
&& self.budget == other.budget
&& self.tie_break == other.tie_break
&& self.restart_policy == other.restart_policy
&& self.children.len() == other.children.len()
&& self
.children
.iter()
.zip(other.children.iter())
.all(|(left, right)| left.spec_eq(right))
}
#[must_use]
pub fn spec_fingerprint(&self) -> u64 {
let mut hasher = crate::util::DetHasher::default();
hasher.write(self.name.as_str().as_bytes());
hash_budget_option(&mut hasher, self.budget);
hash_start_tie_break(&mut hasher, self.tie_break);
hash_restart_policy(&mut hasher, self.restart_policy);
hasher.write_u64(self.children.len() as u64);
for child in &self.children {
hash_child_spec_fields(child, &mut hasher);
}
std::hash::Hasher::finish(&hasher)
}
pub fn compile(self) -> Result<CompiledSupervisor, SupervisorCompileError> {
CompiledSupervisor::new(self)
}
}
fn hash_child_spec_fields(spec: &ChildSpec, hasher: &mut crate::util::DetHasher) {
hasher.write(spec.name.as_str().as_bytes());
hash_supervision_strategy(hasher, &spec.restart);
hash_budget(hasher, spec.shutdown_budget);
hasher.write_u64(spec.depends_on.len() as u64);
for dep in &spec.depends_on {
hasher.write(dep.as_str().as_bytes());
}
hash_registration_policy(hasher, &spec.registration);
hasher.write_u8(u8::from(spec.start_immediately));
hasher.write_u8(u8::from(spec.required));
}
fn hash_budget_option(hasher: &mut crate::util::DetHasher, budget: Option<Budget>) {
match budget {
Some(value) => {
hasher.write_u8(1);
hash_budget(hasher, value);
}
None => hasher.write_u8(0),
}
}
fn hash_budget(hasher: &mut crate::util::DetHasher, budget: Budget) {
match budget.deadline {
Some(deadline) => {
hasher.write_u8(1);
hasher.write_u64(deadline.as_nanos());
}
None => hasher.write_u8(0),
}
hasher.write_u32(budget.poll_quota);
match budget.cost_quota {
Some(cost) => {
hasher.write_u8(1);
hasher.write_u64(cost);
}
None => hasher.write_u8(0),
}
hasher.write_u8(budget.priority);
}
fn hash_supervision_strategy(hasher: &mut crate::util::DetHasher, strategy: &SupervisionStrategy) {
match strategy {
SupervisionStrategy::Stop => hasher.write_u8(0),
SupervisionStrategy::Restart(config) => {
hasher.write_u8(1);
hash_restart_config(hasher, config);
}
SupervisionStrategy::Escalate => hasher.write_u8(2),
}
}
fn duration_nanos_u64(duration: Duration) -> u64 {
u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX)
}
fn hash_restart_config(hasher: &mut crate::util::DetHasher, config: &RestartConfig) {
hasher.write_u32(config.max_restarts);
hasher.write_u64(duration_nanos_u64(config.window));
hash_backoff_strategy(hasher, &config.backoff);
hasher.write_u64(config.restart_cost);
match config.min_remaining_for_restart {
Some(value) => {
hasher.write_u8(1);
hasher.write_u64(duration_nanos_u64(value));
}
None => hasher.write_u8(0),
}
hasher.write_u32(config.min_polls_for_restart);
}
fn hash_backoff_strategy(hasher: &mut crate::util::DetHasher, strategy: &BackoffStrategy) {
match strategy {
BackoffStrategy::None => hasher.write_u8(0),
BackoffStrategy::Fixed(value) => {
hasher.write_u8(1);
hasher.write_u64(duration_nanos_u64(*value));
}
BackoffStrategy::Exponential {
initial,
max,
multiplier,
} => {
hasher.write_u8(2);
hasher.write_u64(duration_nanos_u64(*initial));
hasher.write_u64(duration_nanos_u64(*max));
hasher.write_u64(multiplier.to_bits());
}
}
}
fn hash_registration_policy(hasher: &mut crate::util::DetHasher, policy: &NameRegistrationPolicy) {
match policy {
NameRegistrationPolicy::None => hasher.write_u8(0),
NameRegistrationPolicy::Register { name, collision } => {
hasher.write_u8(1);
hasher.write(name.as_bytes());
hash_collision_policy(hasher, *collision);
}
}
}
fn hash_collision_policy(hasher: &mut crate::util::DetHasher, policy: NameCollisionPolicy) {
match policy {
NameCollisionPolicy::Fail => hasher.write_u8(0),
NameCollisionPolicy::Replace => hasher.write_u8(1),
NameCollisionPolicy::Wait => hasher.write_u8(2),
}
}
fn hash_restart_policy(hasher: &mut crate::util::DetHasher, policy: RestartPolicy) {
match policy {
RestartPolicy::OneForOne => hasher.write_u8(0),
RestartPolicy::OneForAll => hasher.write_u8(1),
RestartPolicy::RestForOne => hasher.write_u8(2),
}
}
fn hash_start_tie_break(hasher: &mut crate::util::DetHasher, tie_break: StartTieBreak) {
match tie_break {
StartTieBreak::InsertionOrder => hasher.write_u8(0),
StartTieBreak::NameLex => hasher.write_u8(1),
}
}
#[derive(Debug)]
pub struct CompiledSupervisor {
pub name: ChildName,
pub budget: Option<Budget>,
pub tie_break: StartTieBreak,
pub restart_policy: RestartPolicy,
pub children: Vec<ChildSpec>,
pub start_order: Vec<usize>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SupervisorRestartPlan {
pub policy: RestartPolicy,
pub cancel_order: Vec<ChildName>,
pub restart_order: Vec<ChildName>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RegionOp {
CancelChild {
name: ChildName,
shutdown_budget: Budget,
},
DrainChild {
name: ChildName,
shutdown_budget: Budget,
},
RestartChild {
name: ChildName,
},
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CompiledRestartOps {
pub policy: RestartPolicy,
pub ops: Vec<RegionOp>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct ReadyKey {
name: ChildName,
idx: usize,
}
impl Ord for ReadyKey {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.name
.cmp(&other.name)
.then_with(|| self.idx.cmp(&other.idx))
}
}
impl PartialOrd for ReadyKey {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
impl CompiledSupervisor {
fn new(builder: SupervisorBuilder) -> Result<Self, SupervisorCompileError> {
let mut name_to_idx = std::collections::HashMap::<ChildName, usize>::new();
for (idx, child) in builder.children.iter().enumerate() {
if name_to_idx.insert(child.name.clone(), idx).is_some() {
return Err(SupervisorCompileError::DuplicateChildName(
child.name.clone(),
));
}
}
let mut indeg = vec![0usize; builder.children.len()];
let mut out = vec![Vec::<usize>::new(); builder.children.len()];
for (idx, child) in builder.children.iter().enumerate() {
let mut seen_deps = std::collections::HashSet::new();
for dep in &child.depends_on {
if !seen_deps.insert(dep) {
continue;
}
let Some(&dep_idx) = name_to_idx.get(dep) else {
return Err(SupervisorCompileError::UnknownDependency {
child: child.name.clone(),
depends_on: dep.clone(),
});
};
if child.start_immediately && !builder.children[dep_idx].start_immediately {
return Err(SupervisorCompileError::DeferredDependency {
child: child.name.clone(),
depends_on: dep.clone(),
});
}
indeg[idx] += 1;
out[dep_idx].push(idx);
}
}
let mut ready = std::collections::BTreeSet::<ReadyKey>::new();
for (idx, child) in builder.children.iter().enumerate() {
if indeg[idx] == 0 {
ready.insert(ReadyKey {
name: child.name.clone(),
idx,
});
}
}
let mut order = Vec::with_capacity(builder.children.len());
while let Some(next) = match builder.tie_break {
StartTieBreak::InsertionOrder => ready
.iter()
.min_by(|a, b| a.idx.cmp(&b.idx).then_with(|| a.name.cmp(&b.name)))
.cloned(),
StartTieBreak::NameLex => ready.iter().next().cloned(),
} {
ready.take(&next);
order.push(next.idx);
for &succ in &out[next.idx] {
indeg[succ] = indeg[succ].saturating_sub(1);
if indeg[succ] == 0 {
ready.insert(ReadyKey {
name: builder.children[succ].name.clone(),
idx: succ,
});
}
}
}
if order.len() != builder.children.len() {
let mut remaining = Vec::new();
for (idx, child) in builder.children.iter().enumerate() {
if indeg[idx] > 0 {
remaining.push(child.name.clone());
}
}
remaining.sort();
return Err(SupervisorCompileError::CycleDetected { remaining });
}
Ok(Self {
name: builder.name,
budget: builder.budget,
tie_break: builder.tie_break,
restart_policy: builder.restart_policy,
children: builder.children,
start_order: order,
})
}
#[must_use]
pub fn restart_plan_for(&self, failed_child: &str) -> Option<SupervisorRestartPlan> {
let failed_idx = self
.children
.iter()
.enumerate()
.find_map(|(idx, child)| (child.name == failed_child).then_some(idx))?;
self.restart_plan_for_idx(failed_idx)
}
#[must_use]
pub fn child_start_pos(&self, child_name: &str) -> Option<usize> {
let child_idx = self
.children
.iter()
.enumerate()
.find_map(|(idx, child)| (child.name == child_name).then_some(idx))?;
self.start_pos_for_child_idx(child_idx)
}
#[must_use]
pub fn child_start_order_names(&self) -> Vec<&str> {
self.start_order
.iter()
.map(|&idx| self.children[idx].name.as_str())
.collect()
}
#[must_use]
pub fn child_stop_order_names(&self) -> Vec<&str> {
self.start_order
.iter()
.rev()
.map(|&idx| self.children[idx].name.as_str())
.collect()
}
#[must_use]
fn start_pos_for_child_idx(&self, child_idx: usize) -> Option<usize> {
self.start_order.iter().position(|&idx| idx == child_idx)
}
#[must_use]
pub fn restart_plan_for_failure<E>(
&self,
failed_child: &str,
outcome: &Outcome<(), E>,
) -> Option<SupervisorRestartPlan> {
let failed_idx = self
.children
.iter()
.enumerate()
.find_map(|(idx, child)| (child.name == failed_child).then_some(idx))?;
if !matches!(outcome, Outcome::Err(_)) {
return None;
}
match self.children[failed_idx].restart {
SupervisionStrategy::Restart(_) => self.restart_plan_for_failure_idx(failed_idx),
SupervisionStrategy::Stop | SupervisionStrategy::Escalate => None,
}
}
#[must_use]
fn affected_positions_for_idx(&self, failed_child_idx: usize) -> Option<Vec<usize>> {
let failed_pos = self.start_pos_for_child_idx(failed_child_idx)?;
let total = self.start_order.len();
let affected_positions = match self.restart_policy {
RestartPolicy::OneForOne => vec![failed_pos],
RestartPolicy::OneForAll => (0..total).collect(),
RestartPolicy::RestForOne => (failed_pos..total).collect(),
}
.into_iter()
.filter(|&pos| {
let child_idx = self.start_order[pos];
let child = &self.children[child_idx];
child.start_immediately || child_idx == failed_child_idx
})
.collect::<Vec<_>>();
(!affected_positions.is_empty()).then_some(affected_positions)
}
#[must_use]
fn restart_plan_for_failure_idx(
&self,
failed_child_idx: usize,
) -> Option<SupervisorRestartPlan> {
let affected_positions = self.affected_positions_for_idx(failed_child_idx)?;
let mut cancel_order = Vec::with_capacity(affected_positions.len());
for &pos in affected_positions.iter().rev() {
cancel_order.push(self.children[self.start_order[pos]].name.clone());
}
let child_index_by_name = self
.children
.iter()
.enumerate()
.map(|(idx, child)| (child.name.as_str(), idx))
.collect::<std::collections::HashMap<_, _>>();
let mut affected_children = vec![false; self.children.len()];
for &pos in &affected_positions {
affected_children[self.start_order[pos]] = true;
}
let mut scheduled_restart = vec![false; self.children.len()];
let mut restart_order = Vec::with_capacity(affected_positions.len());
for &pos in &affected_positions {
let child_idx = self.start_order[pos];
let child = &self.children[child_idx];
if !matches!(child.restart, SupervisionStrategy::Restart(_)) {
continue;
}
let dependencies_restartable = child.depends_on.iter().all(|dependency| {
let dep_idx = *child_index_by_name
.get(dependency.as_str())
.expect("compiled supervisor dependency index missing");
!affected_children[dep_idx] || scheduled_restart[dep_idx]
});
if !dependencies_restartable {
continue;
}
scheduled_restart[child_idx] = true;
restart_order.push(child.name.clone());
}
Some(SupervisorRestartPlan {
policy: self.restart_policy,
cancel_order,
restart_order,
})
}
#[must_use]
fn restart_plan_for_idx(&self, failed_child_idx: usize) -> Option<SupervisorRestartPlan> {
let affected_positions = self.affected_positions_for_idx(failed_child_idx)?;
let mut cancel_order = Vec::with_capacity(affected_positions.len());
let mut restart_order = Vec::with_capacity(affected_positions.len());
for &pos in affected_positions.iter().rev() {
cancel_order.push(self.children[self.start_order[pos]].name.clone());
}
for &pos in &affected_positions {
restart_order.push(self.children[self.start_order[pos]].name.clone());
}
Some(SupervisorRestartPlan {
policy: self.restart_policy,
cancel_order,
restart_order,
})
}
#[must_use]
pub fn compile_restart_ops(&self, plan: &SupervisorRestartPlan) -> CompiledRestartOps {
let child_index_by_name = self
.children
.iter()
.enumerate()
.map(|(idx, child)| (child.name.as_str(), idx))
.collect::<std::collections::HashMap<_, _>>();
let child_by_name = |name: &str| -> Option<&ChildSpec> {
child_index_by_name
.get(name)
.map(|&idx| &self.children[idx])
};
let mut ops = Vec::with_capacity(plan.cancel_order.len() * 2 + plan.restart_order.len());
for name in &plan.cancel_order {
let budget = child_by_name(name).map_or(Budget::INFINITE, |c| c.shutdown_budget);
ops.push(RegionOp::CancelChild {
name: name.clone(),
shutdown_budget: budget,
});
}
for name in &plan.cancel_order {
let budget = child_by_name(name).map_or(Budget::INFINITE, |c| c.shutdown_budget);
ops.push(RegionOp::DrainChild {
name: name.clone(),
shutdown_budget: budget,
});
}
let mut affected_children = vec![false; self.children.len()];
for name in &plan.cancel_order {
if let Some(&child_idx) = child_index_by_name.get(name.as_str()) {
affected_children[child_idx] = true;
}
}
let mut scheduled_restart = vec![false; self.children.len()];
for name in &plan.restart_order {
let Some(&child_idx) = child_index_by_name.get(name.as_str()) else {
continue;
};
let child = &self.children[child_idx];
if !matches!(child.restart, SupervisionStrategy::Restart(_)) {
continue;
}
let dependencies_restartable = child.depends_on.iter().all(|dependency| {
let dep_idx = *child_index_by_name
.get(dependency.as_str())
.expect("compiled supervisor dependency index missing");
!affected_children[dep_idx] || scheduled_restart[dep_idx]
});
if !dependencies_restartable {
continue;
}
scheduled_restart[child_idx] = true;
ops.push(RegionOp::RestartChild { name: name.clone() });
}
CompiledRestartOps {
policy: plan.policy,
ops,
}
}
pub fn spawn(
mut self,
state: &mut RuntimeState,
cx: &crate::cx::Cx,
parent_region: RegionId,
parent_budget: Budget,
) -> Result<SupervisorHandle, SupervisorSpawnError> {
let budget = self.budget.unwrap_or(parent_budget);
let region = state.create_child_region(parent_region, budget)?;
let effective_budget = state
.region(region)
.map_or(budget, crate::record::RegionRecord::budget);
let scope: crate::cx::Scope<'static, crate::types::policy::FailFast> =
crate::cx::Scope::<crate::types::policy::FailFast>::new(region, effective_budget);
#[derive(Clone)]
enum BootState {
NotStarted,
Deferred,
Started,
Failed(SpawnError),
DependencyUnavailable {
dependency_error: Option<SpawnError>,
},
}
fn abort_supervisor_boot(state: &mut RuntimeState, region: RegionId) {
let effects =
state.cancel_request(region, &crate::types::CancelReason::shutdown(), None);
state.defer_cancel_dispatch(effects);
if let Some(r) = state.region(region) {
r.begin_close(None);
}
state.advance_region_state(region);
}
let child_index_by_name = self
.children
.iter()
.enumerate()
.map(|(idx, child)| (child.name.clone(), idx))
.collect::<std::collections::HashMap<_, _>>();
let mut boot_states = vec![BootState::NotStarted; self.children.len()];
let mut started = Vec::new();
for &idx in &self.start_order {
let (child_name, child_required, child_dependencies, start_immediately) = {
let child = &self.children[idx];
(
child.name.clone(),
child.required,
child.depends_on.clone(),
child.start_immediately,
)
};
if !start_immediately {
boot_states[idx] = BootState::Deferred;
continue;
}
let dependency_unavailable = child_dependencies.iter().find_map(|dependency| {
let dep_idx = *child_index_by_name
.get(dependency)
.expect("compiled supervisor dependency index missing");
match &boot_states[dep_idx] {
BootState::Started => None,
BootState::Failed(err) => Some((dependency.clone(), Some(err.clone()))),
BootState::DependencyUnavailable { dependency_error } => {
Some((dependency.clone(), dependency_error.clone()))
}
BootState::NotStarted | BootState::Deferred => Some((dependency.clone(), None)),
}
});
if let Some((dependency, dependency_error)) = dependency_unavailable {
cx.trace("supervisor_child_start_blocked_dependency");
if child_required {
abort_supervisor_boot(state, region);
return Err(SupervisorSpawnError::DependencyUnavailable {
child: child_name,
dependency,
dependency_error,
region,
});
}
boot_states[idx] = BootState::DependencyUnavailable { dependency_error };
continue;
}
let child = &mut self.children[idx];
match child.start.start(&scope, state, cx) {
Ok(task_id) => started.push(StartedChild {
name: child_name.clone(),
task_id,
}),
Err(err) => {
boot_states[idx] = BootState::Failed(err.clone());
cx.trace("supervisor_child_start_failed");
if child_required {
abort_supervisor_boot(state, region);
return Err(SupervisorSpawnError::ChildStartFailed {
child: child_name,
err,
region,
});
}
}
}
if matches!(boot_states[idx], BootState::NotStarted) {
boot_states[idx] = BootState::Started;
}
}
Ok(SupervisorHandle {
name: self.name,
region,
started,
})
}
}
#[derive(Debug)]
pub struct SupervisorHandle {
pub name: ChildName,
pub region: RegionId,
pub started: Vec<StartedChild>,
}
#[derive(Debug)]
pub struct StartedChild {
pub name: ChildName,
pub task_id: TaskId,
}
pub use managed::{
ManagedChildBinding, ManagedChildCompletion, ManagedChildFactory, ManagedChildFuture,
ManagedGeneration, ManagedRestartMode, ManagedSupervisor, ManagedSupervisorBindError,
ManagedSupervisorError, ManagedSupervisorHandle, ManagedSupervisorReport,
};
mod managed {
use super::{
Arc, BTreeMap, Budget, BudgetRefusal, CancelReason, ChildName, ChildSpec,
CompiledSupervisor, Duration, EscalationPolicy, NameRegistrationPolicy, Outcome, RegionId,
RestartPolicy, RestartTracker, RestartVerdict, SpawnError, SupervisionConfig,
SupervisorBuilder, SupervisorCompileError, TaskId, Time,
};
use crate::cx::{ChildRegion, ChildRegionError, ChildRegionSpec, Cx};
use crate::runtime::{JoinError, TaskHandle};
use crate::types::PanicPayload;
use parking_lot::Mutex;
use std::future::{Future, poll_fn};
use std::panic::{AssertUnwindSafe, catch_unwind};
use std::pin::Pin;
use std::task::{Poll, Waker};
const SCAN_QUANTUM: usize = 32;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ManagedRestartMode {
Permanent,
Transient,
Temporary,
}
impl ManagedRestartMode {
fn eligible<E>(self, completed: &ManagedChildCompletion<E>) -> bool {
match self {
Self::Permanent => true,
Self::Transient => {
matches!(completed.outcome, Outcome::Panicked(_))
|| matches!(completed.task_outcome, Err(JoinError::Panicked(_)))
|| (completed.task_outcome.is_ok()
&& matches!(completed.outcome, Outcome::Err(_)))
}
Self::Temporary => false,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ManagedGeneration {
pub number: u64,
pub region: RegionId,
pub task: TaskId,
}
pub type ManagedChildFuture<E> = Pin<Box<dyn Future<Output = Outcome<(), E>> + Send + 'static>>;
pub trait ManagedChildFactory<E>: Send + Sync + 'static {
fn start(&self, cx: Cx, generation: ManagedGeneration) -> ManagedChildFuture<E>;
}
impl<E, F, Fut> ManagedChildFactory<E> for F
where
F: Fn(Cx, ManagedGeneration) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Outcome<(), E>> + Send + 'static,
{
fn start(&self, cx: Cx, generation: ManagedGeneration) -> ManagedChildFuture<E> {
Box::pin(self(cx, generation))
}
}
pub struct ManagedChildBinding<E> {
name: ChildName,
mode: ManagedRestartMode,
factory: Arc<dyn ManagedChildFactory<E>>,
}
impl<E> std::fmt::Debug for ManagedChildBinding<E> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ManagedChildBinding")
.field("name", &self.name)
.field("mode", &self.mode)
.finish_non_exhaustive()
}
}
impl<E> ManagedChildBinding<E> {
#[must_use]
pub fn new(
name: impl Into<ChildName>,
mode: ManagedRestartMode,
factory: impl ManagedChildFactory<E>,
) -> Self {
Self {
name: name.into(),
mode,
factory: Arc::new(factory),
}
}
}
#[derive(Debug)]
#[non_exhaustive]
pub enum ManagedSupervisorBindError {
Topology(SupervisorCompileError),
StartOrder,
Duplicate(ChildName),
Unknown(ChildName),
Missing(ChildName),
RestartPolicyMismatch,
UnsupportedRegistration(ChildName),
InvalidStormThreshold,
}
#[derive(Debug)]
#[non_exhaustive]
pub enum ManagedSupervisorError {
Region(ChildRegionError),
Spawn(SpawnError),
ChildNotStarted {
child: ChildName,
generation: ManagedGeneration,
outcome: Outcome<(), ()>,
},
DependencyUnavailable {
child: ChildName,
dependency: ChildName,
},
RestartLimit {
child: ChildName,
refusal: BudgetRefusal,
},
GenerationExhausted(ChildName),
Cleanup {
child: ChildName,
generation: ManagedGeneration,
outcome: crate::record::task::TaskOutcome,
},
SupervisorCleanup(crate::record::task::TaskOutcome),
Escalation(SpawnError),
}
#[derive(Debug)]
pub struct ManagedChildCompletion<E> {
pub name: ChildName,
pub generation: ManagedGeneration,
pub outcome: Outcome<(), E>,
pub task_outcome: Result<(), JoinError>,
pub shutdown_requested_before_completion: bool,
pub completed_at: Time,
pub region_outcome: Option<crate::record::task::TaskOutcome>,
pub cleanup_outcome: Option<crate::record::task::TaskOutcome>,
}
#[derive(Debug)]
pub struct ManagedSupervisorReport<E> {
pub name: ChildName,
pub region: Option<RegionId>,
pub region_outcome: Option<crate::record::task::TaskOutcome>,
pub cleanup_outcome: Option<crate::record::task::TaskOutcome>,
pub outcome: Outcome<(), ManagedSupervisorError>,
pub children: Vec<ManagedChildCompletion<E>>,
pub started: u64,
pub joined: u64,
pub restart_batches: u64,
pub escalations: u8,
}
pub struct ManagedSupervisor<E> {
name: ChildName,
budget: Option<Budget>,
children: Vec<ChildSpec>,
bindings: Vec<ManagedChildBinding<E>>,
config: SupervisionConfig,
}
impl<E> std::fmt::Debug for ManagedSupervisor<E> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ManagedSupervisor")
.field("name", &self.name)
.field("children", &self.children.len())
.field("config", &self.config)
.finish_non_exhaustive()
}
}
impl CompiledSupervisor {
pub fn bind_managed<E>(
self,
bindings: Vec<ManagedChildBinding<E>>,
config: SupervisionConfig,
) -> Result<ManagedSupervisor<E>, ManagedSupervisorBindError> {
if config.restart_policy != self.restart_policy {
return Err(ManagedSupervisorBindError::RestartPolicyMismatch);
}
if config
.storm_threshold
.is_some_and(|n| !n.is_finite() || n <= 0.0)
{
return Err(ManagedSupervisorBindError::InvalidStormThreshold);
}
let supplied_order = self.start_order;
let compiled = SupervisorBuilder {
name: self.name,
budget: self.budget,
tie_break: self.tie_break,
restart_policy: self.restart_policy,
children: self.children,
}
.compile()
.map_err(ManagedSupervisorBindError::Topology)?;
if supplied_order != compiled.start_order {
return Err(ManagedSupervisorBindError::StartOrder);
}
let mut by_name = BTreeMap::new();
for binding in bindings {
if !compiled.children.iter().any(|c| c.name == binding.name) {
return Err(ManagedSupervisorBindError::Unknown(binding.name));
}
let name = binding.name.clone();
if by_name.insert(name.clone(), binding).is_some() {
return Err(ManagedSupervisorBindError::Duplicate(name));
}
}
let mut children: Vec<_> = compiled.children.into_iter().map(Some).collect();
let mut ordered = Vec::with_capacity(children.len());
let mut factories = Vec::with_capacity(children.len());
for index in compiled.start_order {
let child = children[index]
.take()
.expect("validated unique start order");
if !matches!(child.registration, NameRegistrationPolicy::None) {
return Err(ManagedSupervisorBindError::UnsupportedRegistration(
child.name,
));
}
factories.push(
by_name
.remove(&child.name)
.ok_or_else(|| ManagedSupervisorBindError::Missing(child.name.clone()))?,
);
ordered.push(child);
}
Ok(ManagedSupervisor {
name: compiled.name,
budget: compiled.budget,
children: ordered,
bindings: factories,
config,
})
}
}
struct ChildPublication<E> {
started: bool,
identity: Option<ManagedGeneration>,
terminal: Option<(Time, Outcome<(), E>, bool)>,
shutdown_requested: bool,
waiter: Option<Waker>,
}
struct RunningChild<E> {
number: u64,
region: Option<ChildRegion>,
handle: Option<TaskHandle<()>>,
publication: Arc<Mutex<ChildPublication<E>>>,
shutdown_budget: Budget,
cancellation_sent: bool,
start_observed: bool,
terminal_observed: bool,
}
impl<E> RunningChild<E> {
fn cancel(&mut self) -> Result<(), ChildRegionError> {
if !self.cancellation_sent {
self.cancellation_sent = true;
self.publication.lock().shutdown_requested = true;
if let Some(region) = &self.region {
let mut reason = CancelReason::with_origin(
crate::types::CancelKind::User,
region.region_id(),
region.cx().now(),
)
.with_message("managed supervisor generation drain");
if let Some(handle) = &self.handle {
reason = reason.with_task(handle.task_id());
}
region.cancel_with_budget(reason, self.shutdown_budget)?;
}
}
Ok(())
}
}
impl<E> Drop for RunningChild<E> {
fn drop(&mut self) {
let _ = self.cancel();
if let Some(handle) = &self.handle {
handle.abort();
}
}
}
struct Controller<E> {
supervisor: ManagedSupervisor<E>,
cx: Cx,
root: Option<ChildRegion>,
running: Vec<Option<RunningChild<E>>>,
latest: Vec<Option<ManagedChildCompletion<E>>>,
numbers: Vec<u64>,
ready: Vec<(usize, ManagedGeneration)>,
cancel_waker: Option<crate::cx::cx::CancelWakerToken>,
tracker: RestartTracker,
report: ManagedSupervisorReport<E>,
terminated: Arc<std::sync::atomic::AtomicUsize>,
joins_observed: usize,
}
fn panic_payload(payload: Box<dyn std::any::Any + Send>) -> PanicPayload {
let message = crate::cx::scope::payload_to_string(&payload);
std::mem::forget(payload);
PanicPayload::new(message)
}
impl<E: Send + 'static> Controller<E> {
fn new(supervisor: ManagedSupervisor<E>, cx: &Cx) -> Self {
let count = supervisor.children.len();
let tracker = supervisor.config.restart_tracker();
let report = ManagedSupervisorReport {
name: supervisor.name.clone(),
region: None,
region_outcome: None,
cleanup_outcome: None,
outcome: Outcome::Ok(()),
children: Vec::new(),
started: 0,
joined: 0,
restart_batches: 0,
escalations: 0,
};
Self {
supervisor,
cx: cx.clone(),
root: None,
running: (0..count).map(|_| None).collect(),
latest: (0..count).map(|_| None).collect(),
numbers: vec![0; count],
ready: Vec::new(),
cancel_waker: None,
tracker,
report,
terminated: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
joins_observed: 0,
}
}
fn cancelled(&self) -> bool {
self.cx.checkpoint().is_err()
}
fn record_cancel(&mut self) {
if !matches!(self.report.outcome, Outcome::Panicked(_)) {
self.report.outcome = Outcome::Cancelled(
self.cx
.cancel_reason()
.unwrap_or_else(|| CancelReason::user("managed supervisor cancelled")),
);
}
}
fn record_error(&mut self, error: ManagedSupervisorError) {
let cleanup = match &error {
ManagedSupervisorError::Cleanup { child, .. } => Some(
self.supervisor
.children
.iter()
.position(|spec| &spec.name == child),
),
ManagedSupervisorError::SupervisorCleanup(_) => Some(None),
_ => None,
};
let panic = match &error {
ManagedSupervisorError::Cleanup {
outcome: Outcome::Panicked(payload),
..
}
| ManagedSupervisorError::SupervisorCleanup(Outcome::Panicked(payload)) => {
Some(payload.clone())
}
_ => None,
};
if !matches!(self.report.outcome, Outcome::Panicked(_)) {
self.report.outcome = panic.map_or_else(|| Outcome::Err(error), Outcome::Panicked);
}
if let Some(source) = cleanup {
self.escalate(source);
}
}
fn trace(&self, action: &str, index: usize, identity: ManagedGeneration) {
if let Some(trace) = self.cx.trace_buffer() {
let now = self.cx.now();
let outcome = self.latest[index]
.as_ref()
.filter(|completed| completed.generation == identity)
.map_or("pending", |completed| match completed.outcome {
Outcome::Ok(()) => "ok",
Outcome::Err(_) => "err",
Outcome::Cancelled(_) => "cancelled",
Outcome::Panicked(_) => "panicked",
});
let message = format!(
"managed_supervisor_v1 action={action} supervisor={:?} child={:?} generation={} region={:?} task={:?} outcome={outcome}",
self.supervisor.name,
self.supervisor.children[index].name,
identity.number,
identity.region,
identity.task,
);
trace.record_event(|seq| crate::trace::TraceEvent::user_trace(seq, now, &message));
}
}
fn observe_start(&mut self, index: usize) {
let Some(child) = self.running[index].as_mut() else {
return;
};
let identity = {
let publication = child.publication.lock();
if child.start_observed || !publication.started {
return;
}
child.start_observed = true;
publication
.identity
.expect("started generation has a canonical identity")
};
self.report.started += 1;
self.trace("started", index, identity);
}
fn queue_terminal(&mut self, index: usize) {
let identity = self.latest[index]
.as_ref()
.expect("joined terminal")
.generation;
self.ready.push((index, identity));
}
fn accepts_terminal(&self, index: usize, identity: ManagedGeneration) -> bool {
self.running[index].as_ref().is_some_and(|child| {
child.terminal_observed
&& child.number == identity.number
&& self.latest[index]
.as_ref()
.is_some_and(|completed| completed.generation == identity)
})
}
fn joined(&mut self, index: usize, result: Result<(), JoinError>) {
self.joins_observed = self.joins_observed.saturating_add(1);
self.observe_start(index);
let child = self.running[index]
.as_mut()
.expect("joined owned generation");
child.terminal_observed = true;
let handle = child.handle.take().expect("terminal is consumed once");
let task_outcome = result.clone();
let (identity, terminal, shutdown_requested) = {
let mut publication = child.publication.lock();
(
publication.identity.unwrap_or(ManagedGeneration {
number: child.number,
region: child.region.as_ref().expect("owned region").region_id(),
task: handle.task_id(),
}),
publication.terminal.take(),
publication.shutdown_requested,
)
};
let (completed_at, outcome, shutdown_requested_before_completion) = match result {
Err(JoinError::Panicked(payload)) => {
if let Err(secondary) = catch_unwind(AssertUnwindSafe(|| drop(terminal))) {
std::mem::forget(secondary);
}
(
self.cx.now(),
Outcome::Panicked(payload),
shutdown_requested,
)
}
Err(JoinError::PolledAfterCompletion) => {
unreachable!("managed terminal consumed twice")
}
Ok(()) | Err(JoinError::Cancelled(_)) if terminal.is_some() => {
terminal.expect("checked terminal")
}
Err(JoinError::Cancelled(reason)) => (
self.cx.now(),
Outcome::Cancelled(reason),
shutdown_requested,
),
Ok(()) => (
self.cx.now(),
Outcome::Panicked(PanicPayload::new(
"managed child returned without its terminal publication",
)),
shutdown_requested,
),
};
self.report.joined += 1;
let previous = self.latest[index].replace(ManagedChildCompletion {
name: self.supervisor.children[index].name.clone(),
generation: identity,
completed_at,
outcome,
task_outcome,
shutdown_requested_before_completion,
region_outcome: None,
cleanup_outcome: None,
});
if let Err(payload) = catch_unwind(AssertUnwindSafe(|| drop(previous))) {
self.report.outcome = Outcome::Panicked(panic_payload(payload));
}
self.trace("terminal", index, identity);
}
async fn start(&mut self, index: usize) -> Result<(), ManagedSupervisorError> {
if self.cancelled() {
self.record_cancel();
return Ok(());
}
let number = self.numbers[index].checked_add(1).ok_or_else(|| {
ManagedSupervisorError::GenerationExhausted(
self.supervisor.children[index].name.clone(),
)
})?;
let region = self
.root
.as_ref()
.expect("admitted supervisor region")
.cx()
.open_child_region(ChildRegionSpec::inherit())
.await
.map_err(ManagedSupervisorError::Region)?;
if self.cancelled() {
self.record_cancel();
region
.close()
.await
.map_err(ManagedSupervisorError::Region)?;
return Ok(());
}
let publication = Arc::new(Mutex::new(ChildPublication {
started: false,
identity: None,
terminal: None,
shutdown_requested: false,
waiter: None,
}));
let child_publication = Arc::clone(&publication);
let factory = Arc::clone(&self.supervisor.bindings[index].factory);
let region_id = region.region_id();
let spawn = |tally| {
region.cx().spawn(move |cx| async move {
let _tally = tally;
let identity = ManagedGeneration {
number,
region: region_id,
task: cx.task_id(),
};
child_publication.lock().identity = Some(identity);
let constructed =
catch_unwind(AssertUnwindSafe(|| factory.start(cx.clone(), identity)));
let waiter = {
let mut publication = child_publication.lock();
publication.started = true;
publication.waiter.take()
};
if let Some(waiter) = waiter {
waiter.wake();
}
let outcome = match constructed {
Err(payload) => Outcome::Panicked(panic_payload(payload)),
Ok(future) => {
let mut execution =
Box::pin(crate::cx::scope::CatchUnwind { inner: future });
let returned = execution.as_mut().await;
let retired = catch_unwind(AssertUnwindSafe(|| drop(execution)));
match (returned, retired) {
(Err(payload), retirement) => {
if let Err(secondary) = retirement {
std::mem::forget(secondary);
}
Outcome::Panicked(panic_payload(payload))
}
(Ok(outcome), Err(payload)) => {
if let Err(secondary) =
catch_unwind(AssertUnwindSafe(|| drop(outcome)))
{
std::mem::forget(secondary);
}
Outcome::Panicked(panic_payload(payload))
}
(Ok(outcome), Ok(())) => outcome,
}
}
};
let completed_at = cx.now();
let mut publication = child_publication.lock();
let shutdown_requested = publication.shutdown_requested;
publication.terminal = Some((completed_at, outcome, shutdown_requested));
})
};
let handle = crate::combinator::TerminationTally::track_spawn(&self.terminated, spawn)
.map_err(ManagedSupervisorError::Spawn)?;
self.numbers[index] = number;
self.running[index] = Some(RunningChild {
number,
region: Some(region),
handle: Some(handle),
publication,
shutdown_budget: self.supervisor.children[index].shutdown_budget,
cancellation_sent: false,
start_observed: false,
terminal_observed: false,
});
poll_fn(|poll_cx| {
self.cancel_waker = Some(
self.cx
.refresh_cancel_waker(self.cancel_waker, poll_cx.waker()),
);
if self.cancelled() {
self.record_cancel();
return Poll::Ready(());
}
let child = self.running[index].as_mut().expect("owned child");
if let Poll::Ready(result) = child
.handle
.as_mut()
.expect("unjoined child")
.poll_join(poll_cx)
{
self.joined(index, result);
self.queue_terminal(index);
return Poll::Ready(());
}
let waiter = poll_cx.waker().clone();
let (started, old) = {
let mut publication = child.publication.lock();
(publication.started, publication.waiter.replace(waiter))
};
drop(old);
if started {
self.observe_start(index);
Poll::Ready(())
} else {
Poll::Pending
}
})
.await;
if self.running[index]
.as_ref()
.is_some_and(|child| child.terminal_observed && !child.start_observed)
{
let completed = self.latest[index]
.as_ref()
.expect("observed unstarted terminal");
let outcome = match &completed.outcome {
Outcome::Ok(()) => Outcome::Ok(()),
Outcome::Err(_) => Outcome::Err(()),
Outcome::Cancelled(reason) => Outcome::Cancelled(reason.clone()),
Outcome::Panicked(payload) => Outcome::Panicked(payload.clone()),
};
return Err(ManagedSupervisorError::ChildNotStarted {
child: completed.name.clone(),
generation: completed.generation,
outcome,
});
}
Ok(())
}
async fn drain(&mut self, index: usize) -> Result<(), ManagedSupervisorError> {
if self.running[index].is_none() {
return Ok(());
}
self.running[index]
.as_mut()
.expect("owned child")
.cancel()
.map_err(ManagedSupervisorError::Region)?;
if self.running[index]
.as_ref()
.expect("owned child")
.handle
.is_some()
{
let result = poll_fn(|cx| {
self.running[index]
.as_mut()
.expect("owned child")
.handle
.as_mut()
.expect("unjoined child")
.poll_join(cx)
})
.await;
self.joined(index, result);
}
let mut child = self.running[index].take().expect("owned child");
let region = child.region.take().expect("owned region");
let receipt = region
.close_with_outcome()
.await
.map_err(ManagedSupervisorError::Region)?;
let completed = self.latest[index]
.as_mut()
.expect("joined generation before close");
completed.region_outcome = Some(receipt.outcome);
completed.cleanup_outcome = receipt.cleanup_outcome;
let identity = completed.generation;
let failure = completed
.cleanup_outcome
.as_ref()
.filter(|outcome| !outcome.is_ok())
.map(|outcome| ManagedSupervisorError::Cleanup {
child: completed.name.clone(),
generation: identity,
outcome: outcome.clone(),
});
self.trace("drained", index, identity);
if let Some(error) = failure {
return Err(error);
}
Ok(())
}
fn cancel_children(&mut self, indices: impl Iterator<Item = usize>) -> bool {
let mut accepted = true;
for index in indices {
if let Some(child) = self.running[index].as_mut() {
if let Err(error) = child.cancel() {
accepted = false;
self.record_error(ManagedSupervisorError::Region(error));
}
}
}
accepted
}
fn dependency_unavailable(&self, index: usize) -> Option<ChildName> {
self.supervisor.children[index]
.depends_on
.iter()
.find(|name| {
let dependency = self
.supervisor
.children
.iter()
.position(|child| &child.name == *name)
.expect("validated dependency");
self.running[dependency].is_none()
})
.cloned()
}
async fn wait_exit(&mut self) -> Option<usize> {
let mut cursor: usize = 0;
poll_fn(|poll_cx| {
self.cancel_waker = Some(
self.cx
.refresh_cancel_waker(self.cancel_waker, poll_cx.waker()),
);
if self.cancelled() {
self.record_cancel();
return Poll::Ready(None);
}
if !matches!(self.report.outcome, Outcome::Ok(())) {
return Poll::Ready(None);
}
let end = cursor.saturating_add(SCAN_QUANTUM).min(self.running.len());
while cursor < end {
let index = cursor;
cursor += 1;
if let Some(child) = self.running[index].as_mut() {
if let Some(handle) = &mut child.handle {
if let Poll::Ready(result) = handle.poll_join(poll_cx) {
self.joined(index, result);
self.queue_terminal(index);
}
}
}
}
if cursor < self.running.len() {
poll_cx.waker().wake_by_ref();
return Poll::Pending;
}
cursor = 0;
if !matches!(self.report.outcome, Outcome::Ok(())) {
return Poll::Ready(None);
}
let ready = std::mem::take(&mut self.ready);
self.ready = ready
.into_iter()
.filter(|&(index, identity)| self.accepts_terminal(index, identity))
.collect();
self.ready.sort_by_key(|&(index, _)| {
let completed = self.latest[index].as_ref().expect("current terminal");
(completed.completed_at, completed.generation.task)
});
if !self.ready.is_empty() {
return Poll::Ready(Some(self.ready.remove(0).0));
}
if self.running.iter().all(Option::is_none) {
Poll::Ready(None)
} else {
if self.terminated.load(std::sync::atomic::Ordering::Acquire)
> self.joins_observed
{
poll_cx.waker().wake_by_ref();
}
Poll::Pending
}
})
.await
}
async fn backoff(&mut self, delay: Option<Duration>) -> bool {
if self.cancelled() {
self.record_cancel();
return false;
}
if let Some(delay) = delay.filter(|delay| !delay.is_zero()) {
let mut sleep = Box::pin(crate::time::sleep(self.cx.now(), delay));
let completed = poll_fn(|poll_cx| {
self.cancel_waker = Some(
self.cx
.refresh_cancel_waker(self.cancel_waker, poll_cx.waker()),
);
if self.cancelled() {
self.record_cancel();
return Poll::Ready(false);
}
sleep.as_mut().poll(poll_cx).map(|()| true)
})
.await;
if !completed {
return false;
}
}
if self.cancelled() {
self.record_cancel();
false
} else {
true
}
}
fn escalate(&mut self, source: Option<usize>) {
if self.report.escalations != 0 {
return;
}
let identity = source.map_or(
ManagedGeneration {
number: 0,
region: self.report.region.unwrap_or(self.cx.region_id()),
task: self.cx.task_id(),
},
|index| {
self.latest[index]
.as_ref()
.expect("escalating an observed terminal")
.generation
},
);
let reason = CancelReason::with_origin(
crate::types::CancelKind::FailFast,
identity.region,
self.cx.now(),
)
.with_task(identity.task)
.with_message("managed supervisor restart intensity exhausted");
let result = self
.cx
.spawn_gateway_handle()
.ok_or(SpawnError::RuntimeUnavailable)
.and_then(|gateway| {
gateway.enqueue_region_command(
crate::runtime::spawn_mailbox::RegionCommand::Cancel {
region_id: self.cx.region_id(),
reason,
},
)
});
match result {
Ok(()) => {
self.report.escalations = 1;
if let Some(index) = source {
self.trace("parent_escalated", index, identity);
}
}
Err(error) => {
self.report.outcome = Outcome::Err(ManagedSupervisorError::Escalation(error))
}
}
}
async fn execute(&mut self) {
if self.cancelled() {
self.record_cancel();
return;
}
let mut spec = ChildRegionSpec::inherit();
spec.budget = self.supervisor.budget;
match self.cx.open_child_region(spec).await {
Ok(region) => {
self.report.region = Some(region.region_id());
self.root = Some(region);
}
Err(error) => {
self.report.outcome = Outcome::Err(ManagedSupervisorError::Region(error));
return;
}
}
for index in 0..self.running.len() {
if self.cancelled() {
self.record_cancel();
return;
}
if !self.supervisor.children[index].start_immediately {
continue;
}
if let Some(dependency) = self.dependency_unavailable(index) {
if self.supervisor.children[index].required {
self.report.outcome =
Outcome::Err(ManagedSupervisorError::DependencyUnavailable {
child: self.supervisor.children[index].name.clone(),
dependency,
});
return;
}
continue;
}
if let Err(error) = self.start(index).await {
if self.supervisor.children[index].required {
self.record_error(error);
return;
} else if let Err(cleanup) = self.drain(index).await {
self.record_error(cleanup);
return;
}
}
if !matches!(self.report.outcome, Outcome::Ok(())) {
return;
}
}
while let Some(failed) = self.wait_exit().await {
let eligible = self.supervisor.bindings[failed]
.mode
.eligible(self.latest[failed].as_ref().expect("joined terminal"));
if !eligible {
if let Err(error) = self.drain(failed).await {
self.record_error(error);
return;
}
continue;
}
let now = self.cx.now().as_nanos();
let mut verdict = self.tracker.evaluate_with_budget(now, &self.cx.budget());
if matches!(verdict, RestartVerdict::Denied { .. })
&& self.supervisor.config.escalation == EscalationPolicy::ResetCounter
{
self.tracker.reset();
verdict = self.tracker.evaluate_with_budget(now, &self.cx.budget());
}
let delay = match verdict {
RestartVerdict::Allowed { delay, .. } => delay,
RestartVerdict::Denied { refusal } => {
if self.supervisor.config.escalation == EscalationPolicy::Stop {
if let Err(error) = self.drain(failed).await {
self.record_error(error);
return;
}
continue;
}
self.report.outcome = Outcome::Err(ManagedSupervisorError::RestartLimit {
child: self.supervisor.children[failed].name.clone(),
refusal,
});
if self.supervisor.config.escalation == EscalationPolicy::Escalate {
self.escalate(Some(failed));
}
return;
}
};
let affected: Vec<_> = (0..self.running.len())
.filter(|&index| {
self.running[index].is_some()
&& match self.supervisor.config.restart_policy {
RestartPolicy::OneForOne => index == failed,
RestartPolicy::OneForAll => true,
RestartPolicy::RestForOne => index >= failed,
}
})
.collect();
let cancelled = self.cancel_children(affected.iter().rev().copied());
let mut drained = true;
for &index in affected.iter().rev() {
if let Err(error) = self.drain(index).await {
self.record_error(error);
drained = false;
}
}
if !cancelled || !drained {
return;
}
let restart: Vec<_> = affected
.iter()
.copied()
.filter(|&index| {
let mode = self.supervisor.bindings[index].mode;
let completed = self.latest[index].as_ref().expect("drained generation");
mode != ManagedRestartMode::Temporary
&& (completed.shutdown_requested_before_completion
|| mode.eligible(completed))
})
.collect();
if !self.backoff(delay).await {
return;
}
let mut counted = false;
for index in restart {
if self.dependency_unavailable(index).is_some() {
continue;
}
if self.cancelled() {
self.record_cancel();
return;
}
if !counted {
self.tracker.record(self.cx.now().as_nanos());
self.report.restart_batches += 1;
counted = true;
}
if let Err(error) = self.start(index).await {
if self.supervisor.children[index].required {
self.record_error(error);
return;
} else if let Err(cleanup) = self.drain(index).await {
self.record_error(cleanup);
return;
}
}
if !matches!(self.report.outcome, Outcome::Ok(())) {
return;
}
}
}
}
async fn finish(&mut self) {
self.cancel_children((0..self.running.len()).rev());
for index in (0..self.running.len()).rev() {
if let Err(error) = self.drain(index).await {
self.record_error(error);
}
}
if let Some(region) = self.root.take() {
match region.close_with_outcome().await {
Err(error) => self.record_error(ManagedSupervisorError::Region(error)),
Ok(receipt) => {
self.report.region_outcome = Some(receipt.outcome);
self.report.cleanup_outcome = receipt.cleanup_outcome;
let failure = self
.report
.cleanup_outcome
.as_ref()
.filter(|outcome| !outcome.is_ok())
.cloned();
if let Some(outcome) = failure {
self.record_error(ManagedSupervisorError::SupervisorCleanup(outcome));
}
}
}
}
if matches!(self.report.outcome, Outcome::Ok(())) && self.cancelled() {
self.record_cancel();
}
if let Some(token) = self.cancel_waker.take() {
self.cx.clear_cancel_waker(token);
}
self.report
.children
.extend(self.latest.iter_mut().filter_map(Option::take));
}
}
impl<E> Drop for Controller<E> {
fn drop(&mut self) {
if let Some(token) = self.cancel_waker.take() {
self.cx.clear_cancel_waker(token);
}
for child in self.running.iter_mut().rev().flatten() {
let _ = child.cancel();
}
}
}
pub struct ManagedSupervisorHandle<E> {
task: TaskHandle<()>,
report: Arc<Mutex<Option<ManagedSupervisorReport<E>>>>,
}
impl<E> std::fmt::Debug for ManagedSupervisorHandle<E> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ManagedSupervisorHandle")
.field("task", &self.task.task_id())
.finish_non_exhaustive()
}
}
impl<E> ManagedSupervisorHandle<E> {
#[must_use]
pub fn task_id(&self) -> TaskId {
self.task.task_id()
}
pub fn abort(&self) {
self.task.abort();
}
pub async fn join(&mut self) -> Result<ManagedSupervisorReport<E>, JoinError> {
let terminal = poll_fn(|cx| self.task.poll_join(cx)).await;
if let Some(mut report) = self.report.lock().take() {
if let Err(JoinError::Panicked(payload)) = terminal {
report.outcome = Outcome::Panicked(payload);
}
return Ok(report);
}
match terminal {
Err(error) => Err(error),
Ok(()) => Err(JoinError::Panicked(PanicPayload::new(
"managed controller omitted its report",
))),
}
}
}
impl<E> Drop for ManagedSupervisorHandle<E> {
fn drop(&mut self) {
self.task.abort();
}
}
impl<E: Send + 'static> ManagedSupervisor<E> {
pub async fn run(self, cx: &Cx) -> ManagedSupervisorReport<E> {
let mut controller = Controller::new(self, cx);
controller.execute().await;
controller.finish().await;
let empty = ManagedSupervisorReport {
name: controller.report.name.clone(),
region: controller.report.region,
region_outcome: None,
cleanup_outcome: None,
outcome: Outcome::Ok(()),
children: Vec::new(),
started: 0,
joined: 0,
restart_batches: 0,
escalations: 0,
};
std::mem::replace(&mut controller.report, empty)
}
pub fn spawn(self, cx: &Cx) -> Result<ManagedSupervisorHandle<E>, SpawnError> {
let report = Arc::new(Mutex::new(None));
let publication = Arc::clone(&report);
let task = cx.spawn(move |controller_cx| async move {
let result = self.run(&controller_cx).await;
*publication.lock() = Some(result);
})?;
Ok(ManagedSupervisorHandle { task, report })
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::pedantic, clippy::nursery, clippy::future_not_send)]
use super::super::{BackoffStrategy, NameCollisionPolicy, RuntimeState};
use super::*;
use crate::channel::{mpsc, oneshot};
use crate::lab::{LabConfig, LabRuntime};
use std::sync::atomic::{AtomicUsize, Ordering};
fn legacy_must_not_run(
_: &crate::cx::Scope<'static, crate::types::policy::FailFast>,
_: &mut RuntimeState,
_: &Cx,
) -> Result<TaskId, SpawnError> {
panic!("managed binding must not invoke a consumed legacy ChildStart")
}
fn forbidden_generation(
_: Cx,
_: ManagedGeneration,
) -> std::future::Ready<Outcome<(), ()>> {
panic!("second child must never be started after parent cancellation")
}
fn topology(names: &[&str], policy: RestartPolicy) -> CompiledSupervisor {
let mut builder = SupervisorBuilder::new("managed-test").with_restart_policy(policy);
for name in names {
builder = builder.child(
ChildSpec::new(*name, legacy_must_not_run)
.with_shutdown_budget(Budget::new().with_poll_quota(17)),
);
}
builder.compile().unwrap()
}
fn config(policy: RestartPolicy, restarts: u32) -> SupervisionConfig {
SupervisionConfig::new(restarts, Duration::from_secs(60))
.with_restart_policy(policy)
.with_backoff(BackoffStrategy::None)
}
fn clean(lab: &mut LabRuntime, root: RegionId) {
assert_eq!(lab.state.live_task_count(), 0);
assert_eq!(lab.state.pending_obligation_count(), 0);
assert!(lab.run_until_quiescent_with_report().lab_test_passed());
if lab.state.region(root).is_some() {
let (tasks, wakes) = lab
.state
.cancel_request(root, &CancelReason::user("managed test finished"), None)
.into_parts();
assert!(tasks.is_empty());
wakes.dispatch();
lab.state.advance_region_state(root);
}
assert!(lab.state.region(root).is_none());
assert!(lab.run_until_quiescent_with_report().lab_test_passed());
}
fn run_case<F, Fut, T>(factory: F) -> T
where
F: FnOnce(Cx) -> Fut + Send + 'static,
Fut: Future<Output = T> + Send + 'static,
T: Send + 'static,
{
let mut lab = LabRuntime::new(LabConfig::new(0x34_0001).max_steps(8192));
let root = lab.state.create_root_region(Budget::INFINITE);
let (task, mut join) = lab
.state
.create_task(root, Budget::INFINITE, async move {
factory(Cx::current().expect("registered managed controller")).await
})
.unwrap();
lab.scheduler.lock().schedule(task, 0);
lab.run_until_idle();
let result = join
.try_join()
.unwrap()
.expect("actual managed controller finished");
clean(&mut lab, root);
result
}
#[test]
fn managed_real_generations_cover_all_restart_modes_and_outcomes() {
for mode in [
ManagedRestartMode::Permanent,
ManagedRestartMode::Transient,
ManagedRestartMode::Temporary,
] {
for first in 0..6 {
let report = run_case(move |cx| async move {
let binding = ManagedChildBinding::new(
"child",
mode,
move |_child: Cx, generation: ManagedGeneration| {
if first == 4 && generation.number == 1 {
panic!("actual factory panic");
}
async move {
if generation.number > 1 {
return Outcome::Ok(());
}
match first {
0 => Outcome::Ok(()),
1 => Outcome::Err(String::from("domain failure")),
2 => Outcome::Cancelled(CancelReason::user(
"child finished cancelled",
)),
3 => Outcome::Panicked(PanicPayload::new("encoded panic")),
5 => panic!("actual child future poll panic"),
_ => unreachable!(),
}
}
},
);
let managed = topology(&["child"], RestartPolicy::OneForOne)
.bind_managed(vec![binding], config(RestartPolicy::OneForOne, 1))
.unwrap();
let mut handle = managed.spawn(&cx).unwrap();
let report = handle.join().await.unwrap();
assert_ne!(handle.task_id(), report.children[0].generation.task);
report
});
let restarted = mode == ManagedRestartMode::Permanent
|| (mode == ManagedRestartMode::Transient
&& matches!(first, 1 | 3 | 4 | 5));
assert!(matches!(report.outcome, Outcome::Ok(())));
assert_eq!(report.started, 1 + u64::from(restarted));
assert_eq!(report.joined, report.started);
assert_eq!(report.restart_batches, u64::from(restarted));
assert_eq!(report.escalations, 0);
assert_eq!(report.children.len(), 1);
assert_eq!(report.children[0].generation.number, report.started);
if restarted || first == 0 {
assert!(report.children[0].outcome.is_ok());
} else if first == 1 {
assert!(
matches!(&report.children[0].outcome, Outcome::Err(error) if error == "domain failure")
);
} else if first == 2 {
assert!(report.children[0].outcome.is_cancelled());
} else {
assert!(report.children[0].outcome.is_panicked());
}
}
}
}
#[test]
fn managed_empty_topology_and_send_only_error_need_no_fake_child() {
let report = run_case(|cx| async move {
let managed = topology(&[], RestartPolicy::OneForOne)
.bind_managed(
Vec::<ManagedChildBinding<std::cell::Cell<u8>>>::new(),
config(RestartPolicy::OneForOne, 1),
)
.unwrap();
managed.run(&cx).await
});
assert!(report.outcome.is_ok());
assert_eq!(report.started, 0);
assert_eq!(report.joined, 0);
assert!(report.region.is_some());
assert!(report.children.is_empty());
}
type StartedLog = Arc<Mutex<Vec<(String, ManagedGeneration, mpsc::Sender<()>)>>>;
fn parked_binding(
name: &'static str,
log: StartedLog,
) -> ManagedChildBinding<&'static str> {
ManagedChildBinding::new(
name,
ManagedRestartMode::Transient,
move |cx: Cx, generation| {
let (sender, mut receiver) = mpsc::channel(1);
log.lock().push((name.to_string(), generation, sender));
async move {
match receiver.recv(&cx).await {
Ok(()) => Outcome::Err("triggered child failure"),
Err(_) => Outcome::Cancelled(
cx.cancel_reason().expect("real region cancellation"),
),
}
}
},
)
}
#[test]
fn managed_three_strategies_drain_actual_finalizers_before_replacement() {
for policy in [
RestartPolicy::OneForOne,
RestartPolicy::OneForAll,
RestartPolicy::RestForOne,
] {
let mut lab = LabRuntime::new(LabConfig::new(0x34_0002).max_steps(8192));
let root = lab.state.create_root_region(Budget::INFINITE);
let log: StartedLog = Arc::new(Mutex::new(Vec::new()));
let bindings = ["a", "b", "c"]
.into_iter()
.map(|name| parked_binding(name, Arc::clone(&log)))
.collect();
let managed = topology(&["a", "b", "c"], policy)
.bind_managed(bindings, config(policy, 4))
.unwrap();
let result = Arc::new(Mutex::new(None));
let publication = Arc::clone(&result);
let (parent, mut join) = lab
.state
.create_task(root, Budget::INFINITE, async move {
let cx = Cx::current().unwrap();
*publication.lock() = Some(managed.run(&cx).await);
})
.unwrap();
let parent_cx = lab.state.task(parent).unwrap().cx.clone().unwrap();
lab.scheduler.lock().schedule(parent, 0);
lab.run_until_idle();
assert_eq!(
log.lock()
.iter()
.map(|entry| entry.0.as_str())
.collect::<Vec<_>>(),
["a", "b", "c"]
);
let old_b = log.lock()[1].1;
let (release, mut wait) = oneshot::channel();
let finalizer_polled = Arc::new(AtomicUsize::new(0));
let finalizer_done = Arc::new(AtomicUsize::new(0));
let polled = Arc::clone(&finalizer_polled);
let done = Arc::clone(&finalizer_done);
assert!(
lab.state
.register_async_finalizer(old_b.region, async move {
polled.fetch_add(1, Ordering::SeqCst);
wait.recv_uninterruptible().await.unwrap();
done.fetch_add(1, Ordering::SeqCst);
})
);
log.lock()[1].2.try_send(()).unwrap();
lab.run_until_idle();
assert_eq!(finalizer_polled.load(Ordering::SeqCst), 1);
assert_eq!(finalizer_done.load(Ordering::SeqCst), 0);
assert_eq!(
log.lock().len(),
3,
"replacement cannot start while an old region finalizer is Pending"
);
assert!(result.lock().is_none());
assert!(join.try_join().unwrap().is_none());
release.send(&parent_cx, ()).unwrap();
lab.run_until_idle();
let names: Vec<_> = log
.lock()
.iter()
.skip(3)
.map(|entry| entry.0.clone())
.collect();
let expected: &[&str] = match policy {
RestartPolicy::OneForOne => &["b"],
RestartPolicy::OneForAll => &["a", "b", "c"],
RestartPolicy::RestForOne => &["b", "c"],
};
assert_eq!(names, expected);
assert_eq!(finalizer_done.load(Ordering::SeqCst), 1);
assert!(lab.state.region(old_b.region).is_none());
for (_, replacement, _) in log.lock().iter().skip(3) {
assert_eq!(replacement.number, 2);
assert_ne!(replacement.region, old_b.region);
assert!(lab.state.task(replacement.task).is_some());
}
join.abort();
lab.run_until_idle();
assert!(matches!(join.try_join(), Err(JoinError::Cancelled(_))));
let report = result
.lock()
.take()
.expect("cancelled controller drains then publishes report");
assert!(report.outcome.is_cancelled());
assert_eq!(report.restart_batches, 1);
assert_eq!(report.started, (3 + expected.len()) as u64);
assert_eq!(report.joined, report.started);
assert_eq!(report.children.len(), 3);
clean(&mut lab, root);
}
}
#[test]
fn managed_parent_cancellation_during_backoff_forbids_resurrection() {
let mut lab = LabRuntime::new(LabConfig::new(0x34_0003).max_steps(4096));
let root = lab.state.create_root_region(Budget::INFINITE);
let started = Arc::new(AtomicUsize::new(0));
let counter = Arc::clone(&started);
let binding = ManagedChildBinding::new(
"child",
ManagedRestartMode::Permanent,
move |_: Cx, _| {
counter.fetch_add(1, Ordering::SeqCst);
async { Outcome::<(), ()>::Err(()) }
},
);
let managed = topology(&["child"], RestartPolicy::OneForOne)
.bind_managed(
vec![binding],
config(RestartPolicy::OneForOne, 4)
.with_backoff(BackoffStrategy::Fixed(Duration::from_secs(30))),
)
.unwrap();
let result = Arc::new(Mutex::new(None));
let publication = Arc::clone(&result);
let (parent, mut join) = lab
.state
.create_task(root, Budget::INFINITE, async move {
*publication.lock() = Some(managed.run(&Cx::current().unwrap()).await);
})
.unwrap();
lab.scheduler.lock().schedule(parent, 0);
lab.run_until_idle();
assert_eq!(started.load(Ordering::SeqCst), 1);
assert!(result.lock().is_none());
assert!(join.try_join().unwrap().is_none());
join.abort();
lab.run_until_idle();
assert!(matches!(join.try_join(), Err(JoinError::Cancelled(_))));
let report = result.lock().take().unwrap();
assert!(report.outcome.is_cancelled());
assert_eq!(report.started, 1);
assert_eq!(report.joined, 1);
assert_eq!(report.restart_batches, 0);
assert_eq!(started.load(Ordering::SeqCst), 1);
assert!(matches!(report.children[0].outcome, Outcome::Err(())));
clean(&mut lab, root);
}
#[test]
fn managed_intensity_exhaustion_cancels_actual_parent_region_once() {
let mut lab = LabRuntime::new(LabConfig::new(0x34_0004).max_steps(8192));
let root = lab.state.create_root_region(Budget::INFINITE);
let (sender, mut receiver) = mpsc::channel::<()>(1);
let cancelled_sibling = Arc::new(AtomicUsize::new(0));
let witnessed = Arc::clone(&cancelled_sibling);
let (sibling, mut sibling_join) = lab
.state
.create_task(root, Budget::INFINITE, async move {
let cx = Cx::current().unwrap();
assert!(receiver.recv(&cx).await.is_err());
assert!(cx.cancel_reason().is_some());
witnessed.fetch_add(1, Ordering::SeqCst);
})
.unwrap();
lab.scheduler.lock().schedule(sibling, 0);
lab.run_until_idle();
assert_eq!(sender.telemetry_snapshot(0).recv_waiter_count, 1);
let bindings = ["a", "b"]
.into_iter()
.map(|name| {
ManagedChildBinding::new(
name,
ManagedRestartMode::Permanent,
|_: Cx, _| async { Outcome::<(), ()>::Err(()) },
)
})
.collect();
let managed = topology(&["a", "b"], RestartPolicy::OneForOne)
.bind_managed(
bindings,
config(RestartPolicy::OneForOne, 1).with_escalation(EscalationPolicy::Escalate),
)
.unwrap();
let result = Arc::new(Mutex::new(None));
let publication = Arc::clone(&result);
let (parent, mut join) = lab
.state
.create_task(root, Budget::INFINITE, async move {
*publication.lock() = Some(managed.run(&Cx::current().unwrap()).await);
})
.unwrap();
lab.scheduler.lock().schedule(parent, 0);
lab.run_until_idle();
let report = result
.lock()
.take()
.expect("parent escalation still drains controller children");
assert!(matches!(
report.outcome,
Outcome::Err(ManagedSupervisorError::RestartLimit { .. })
));
assert_eq!(report.escalations, 1);
assert_eq!(
report.restart_batches, 1,
"two failing children share one allowance"
);
assert_eq!(report.started, 3);
assert_eq!(report.joined, 3);
assert_eq!(cancelled_sibling.load(Ordering::SeqCst), 1);
assert_eq!(sender.telemetry_snapshot(0).recv_waiter_count, 0);
assert!(matches!(
sibling_join.try_join(),
Err(JoinError::Cancelled(_))
));
assert!(matches!(
join.try_join(),
Ok(Some(())) | Err(JoinError::Cancelled(_))
));
let events = lab.state.trace_handle().snapshot();
assert_eq!(events.iter().filter(|event| matches!(&event.data,
crate::trace::TraceData::Message(message) if message.contains("action=parent_escalated"))).count(), 1);
clean(&mut lab, root);
}
#[test]
fn managed_old_generation_event_cannot_drain_a_live_replacement() {
run_case(|cx| async move {
let (sender, receiver) = mpsc::channel::<()>(1);
let receiver = Arc::new(Mutex::new(Some(receiver)));
let held = Arc::clone(&receiver);
let binding = ManagedChildBinding::new(
"child",
ManagedRestartMode::Transient,
move |child: Cx, generation: ManagedGeneration| {
let receiver = (generation.number > 1).then(|| held.lock().take().unwrap());
async move {
let Some(mut receiver) = receiver else {
return Outcome::Err(());
};
assert!(receiver.recv(&child).await.is_err());
Outcome::Cancelled(child.cancel_reason().unwrap())
}
},
);
let managed = topology(&["child"], RestartPolicy::OneForOne)
.bind_managed(vec![binding], config(RestartPolicy::OneForOne, 2))
.unwrap();
let mut controller = Controller::new(managed, &cx);
controller.root = Some(
cx.open_child_region(ChildRegionSpec::inherit())
.await
.unwrap(),
);
controller.start(0).await.unwrap();
assert_eq!(controller.wait_exit().await, Some(0));
let old = controller.latest[0].as_ref().unwrap().generation;
controller.drain(0).await.unwrap();
controller.start(0).await.unwrap();
controller.ready.push((0, old));
assert!(!controller.accepts_terminal(0, old));
let mut wait = Box::pin(controller.wait_exit());
poll_fn(|poll_cx| {
assert!(wait.as_mut().poll(poll_cx).is_pending());
Poll::Ready(())
})
.await;
drop(wait);
assert_eq!(
sender.telemetry_snapshot(0).recv_waiter_count,
1,
"stale event neither cancels nor consumes replacement"
);
assert_eq!(controller.running[0].as_ref().unwrap().number, 2);
controller.finish().await;
assert_eq!(controller.report.joined, 2);
assert_eq!(controller.report.children[0].generation.number, 2);
assert_eq!(sender.telemetry_snapshot(0).recv_waiter_count, 0);
});
}
#[test]
fn managed_binding_refuses_missing_duplicate_and_unimplemented_registration() {
let binding = || {
ManagedChildBinding::new("child", ManagedRestartMode::Temporary, |_: Cx, _| async {
Outcome::<(), ()>::Ok(())
})
};
assert!(matches!(
topology(&["child"], RestartPolicy::OneForOne).bind_managed(
Vec::<ManagedChildBinding<()>>::new(),
config(RestartPolicy::OneForOne, 1)
),
Err(ManagedSupervisorBindError::Missing(_))
));
assert!(matches!(
topology(&["child"], RestartPolicy::OneForOne).bind_managed(
vec![binding(), binding()],
config(RestartPolicy::OneForOne, 1)
),
Err(ManagedSupervisorBindError::Duplicate(_))
));
let mut compiled = topology(&["child"], RestartPolicy::OneForOne);
compiled.children[0].registration = NameRegistrationPolicy::Register {
name: "actual-registry-required".to_string(),
collision: NameCollisionPolicy::Fail,
};
assert!(matches!(
compiled.bind_managed(vec![binding()], config(RestartPolicy::OneForOne, 1)),
Err(ManagedSupervisorBindError::UnsupportedRegistration(_))
));
}
#[test]
fn managed_cancellation_inside_factory_stops_next_start_and_joins_pending_cleanup() {
let mut lab = LabRuntime::new(LabConfig::new(0x34_0005).max_steps(8192));
let root = lab.state.create_root_region(Budget::INFINITE);
let started = Arc::new(AtomicUsize::new(0));
let cleanup_started = Arc::new(AtomicUsize::new(0));
let cleanup_finished = Arc::new(AtomicUsize::new(0));
let identity = Arc::new(Mutex::new(None));
let (release, receiver) = oneshot::channel::<()>();
let receiver = Arc::new(Mutex::new(Some(receiver)));
let output = Arc::new(Mutex::new(None));
let published = Arc::clone(&output);
let factory_started = Arc::clone(&started);
let child_entered = Arc::clone(&cleanup_started);
let child_finished = Arc::clone(&cleanup_finished);
let child_identity = Arc::clone(&identity);
let (parent, mut join) = lab
.state
.create_task(root, Budget::INFINITE, async move {
let parent_cx = Cx::current().unwrap();
let cancel_parent = parent_cx.clone();
let first = ManagedChildBinding::new(
"a",
ManagedRestartMode::Permanent,
move |child: Cx, generation| {
factory_started.fetch_add(1, Ordering::SeqCst);
*child_identity.lock() = Some(generation);
let mut cleanup = receiver.lock().take().unwrap();
let entered = Arc::clone(&child_entered);
let finished = Arc::clone(&child_finished);
cancel_parent.cancel_with(
crate::types::CancelKind::User,
Some("cancel during actual start"),
);
async move {
let (_keep_sender, mut receiver) = mpsc::channel::<()>(1);
assert!(receiver.recv(&child).await.is_err());
entered.fetch_add(1, Ordering::SeqCst);
cleanup.recv_uninterruptible().await.unwrap();
finished.fetch_add(1, Ordering::SeqCst);
Outcome::<(), ()>::Cancelled(child.cancel_reason().unwrap())
}
},
);
let second = ManagedChildBinding::new(
"b",
ManagedRestartMode::Permanent,
forbidden_generation,
);
let managed = topology(&["a", "b"], RestartPolicy::OneForAll)
.bind_managed(vec![first, second], config(RestartPolicy::OneForAll, 4))
.unwrap();
*published.lock() = Some(managed.run(&parent_cx).await);
})
.unwrap();
lab.scheduler.lock().schedule(parent, 0);
lab.run_until_idle();
assert_eq!(started.load(Ordering::SeqCst), 1);
assert_eq!(cleanup_started.load(Ordering::SeqCst), 1);
assert_eq!(cleanup_finished.load(Ordering::SeqCst), 0);
assert!(
output.lock().is_none(),
"controller must retain its actual Pending child"
);
assert!(join.try_join().unwrap().is_none());
let generation = identity.lock().unwrap();
let task = lab
.state
.task(generation.task)
.expect("cleanup child remains runtime-owned");
let cleanup_budget = task
.cleanup_budget()
.expect("real cancellation state installed");
assert!(
cleanup_budget.poll_quota <= 17,
"compiled budget must constrain the actual task, got {cleanup_budget:?}"
);
release.send_blocking(()).unwrap();
lab.run_until_idle();
assert!(matches!(join.try_join(), Err(JoinError::Cancelled(_))));
let report = output
.lock()
.take()
.expect("drained cancelled controller report");
assert!(report.outcome.is_cancelled());
assert_eq!(report.started, 1);
assert_eq!(report.joined, 1);
assert_eq!(report.restart_batches, 0);
assert_eq!(cleanup_finished.load(Ordering::SeqCst), 1);
assert!(lab.state.region(generation.region).is_none());
clean(&mut lab, root);
}
#[test]
fn managed_one_for_all_does_not_resurrect_completed_transient_or_temporary_children() {
let starts = Arc::new(Mutex::new(Vec::new()));
let observed = Arc::clone(&starts);
let report = run_case(move |cx| async move {
let mut bindings = Vec::new();
for (name, mode) in [
("a", ManagedRestartMode::Transient),
("b", ManagedRestartMode::Transient),
("c", ManagedRestartMode::Temporary),
] {
let log = Arc::clone(&starts);
bindings.push(ManagedChildBinding::new(
name,
mode,
move |_: Cx, generation: ManagedGeneration| {
log.lock().push((name, generation.number));
async move {
if name == "a" && generation.number == 1 {
Outcome::Err(())
} else {
Outcome::Ok(())
}
}
},
));
}
topology(&["a", "b", "c"], RestartPolicy::OneForAll)
.bind_managed(bindings, config(RestartPolicy::OneForAll, 1))
.unwrap()
.run(&cx)
.await
});
assert_eq!(*observed.lock(), [("a", 1), ("b", 1), ("c", 1), ("a", 2)]);
assert_eq!(report.started, 4);
assert_eq!(report.joined, 4);
assert_eq!(report.restart_batches, 1);
assert!(report.children.iter().all(|child| child.outcome.is_ok()));
}
#[test]
fn managed_shared_window_boundary_uses_actual_runtime_time() {
for elapsed in [10_u64, 11] {
let mut lab = LabRuntime::new(LabConfig::new(0x34_0006).max_steps(8192));
let root = lab.state.create_root_region(Budget::INFINITE);
let trigger = Arc::new(Mutex::new(None));
let published_trigger = Arc::clone(&trigger);
let binding = ManagedChildBinding::new(
"child",
ManagedRestartMode::Transient,
move |child: Cx, generation: ManagedGeneration| {
let receiver = if generation.number == 2 {
let (sender, receiver) = mpsc::channel::<()>(1);
*published_trigger.lock() = Some(sender);
Some(receiver)
} else {
None
};
async move {
if generation.number == 1 {
return Outcome::Err(());
}
if let Some(mut receiver) = receiver {
receiver.recv(&child).await.unwrap();
return Outcome::Err(());
}
Outcome::Ok(())
}
},
);
let managed = topology(&["child"], RestartPolicy::OneForOne)
.bind_managed(
vec![binding],
SupervisionConfig::new(1, Duration::from_nanos(10))
.with_backoff(BackoffStrategy::None),
)
.unwrap();
let (parent, mut join) = lab
.state
.create_task(root, Budget::INFINITE, async move {
managed.run(&Cx::current().unwrap()).await
})
.unwrap();
lab.scheduler.lock().schedule(parent, 0);
lab.run_until_idle();
assert!(join.try_join().unwrap().is_none());
let sender = trigger
.lock()
.take()
.expect("actual second generation admitted");
assert_eq!(sender.telemetry_snapshot(0).recv_waiter_count, 1);
lab.advance_time(elapsed);
sender.try_send(()).unwrap();
lab.run_until_idle();
let report = join
.try_join()
.unwrap()
.expect("window decision reaches terminal controller");
let expired = elapsed == 11;
assert_eq!(report.restart_batches, 1 + u64::from(expired));
assert_eq!(report.started, 2 + u64::from(expired));
assert_eq!(report.joined, report.started);
assert_eq!(report.children[0].generation.number, report.started);
assert_eq!(report.children[0].outcome.is_ok(), expired);
assert_eq!(report.children[0].completed_at.as_nanos(), elapsed);
assert!(report.children[0].region_outcome.is_some());
clean(&mut lab, root);
}
}
#[test]
fn managed_actual_finalizer_panic_forbids_replacement_and_escalates_once() {
let mut lab = LabRuntime::new(LabConfig::new(0x34_0007).max_steps(8192));
let root = lab.state.create_root_region(Budget::INFINITE);
let log: StartedLog = Arc::new(Mutex::new(Vec::new()));
let binding = parked_binding("child", Arc::clone(&log));
let managed = topology(&["child"], RestartPolicy::OneForOne)
.bind_managed(vec![binding], config(RestartPolicy::OneForOne, 8))
.unwrap();
let output = Arc::new(Mutex::new(None));
let published = Arc::clone(&output);
let (parent, mut join) = lab
.state
.create_task(root, Budget::INFINITE, async move {
*published.lock() = Some(managed.run(&Cx::current().unwrap()).await);
})
.unwrap();
lab.scheduler.lock().schedule(parent, 0);
lab.run_until_idle();
assert_eq!(log.lock().len(), 1);
let generation = log.lock()[0].1;
let finalized = Arc::new(AtomicUsize::new(0));
let counter = Arc::clone(&finalized);
assert!(
lab.state
.register_async_finalizer(generation.region, async move {
counter.fetch_add(1, Ordering::SeqCst);
panic!("actual managed finalizer failure");
})
);
log.lock()[0].2.try_send(()).unwrap();
lab.run_until_idle();
assert!(matches!(
join.try_join(),
Ok(Some(())) | Err(JoinError::Cancelled(_))
));
let report = output
.lock()
.take()
.expect("failed cleanup still publishes an honest terminal receipt");
assert!(report.outcome.is_panicked(), "{report:?}");
assert_eq!(report.escalations, 1);
assert_eq!(report.started, 1);
assert_eq!(report.joined, 1);
assert_eq!(report.restart_batches, 0);
assert_eq!(
log.lock().len(),
1,
"quiescent but failed cleanup cannot authorize replacement"
);
assert_eq!(finalized.load(Ordering::SeqCst), 1);
assert!(matches!(
report.children[0].outcome,
Outcome::Err("triggered child failure")
));
assert!(matches!(
report.children[0].cleanup_outcome,
Some(Outcome::Panicked(_))
));
assert!(report.children[0].region_outcome.is_some());
assert!(lab.state.region(generation.region).is_none());
clean(&mut lab, root);
}
#[test]
fn managed_parent_cancel_during_root_finalizer_precedes_success_publication() {
for cancel_before_close in [true, false] {
let mut lab = LabRuntime::new(LabConfig::new(0x34_0011).max_steps(8192));
let root = lab.state.create_root_region(Budget::INFINITE);
let started: StartedLog = Arc::new(Mutex::new(Vec::new()));
let child_started = Arc::clone(&started);
let binding = ManagedChildBinding::new(
"child",
ManagedRestartMode::Temporary,
move |child: Cx, generation: ManagedGeneration| {
let (sender, mut receiver) = mpsc::channel::<()>(1);
child_started
.lock()
.push(("child".to_owned(), generation, sender));
async move {
receiver.recv(&child).await.unwrap();
Outcome::<(), ()>::Ok(())
}
},
);
let managed = topology(&["child"], RestartPolicy::OneForOne)
.bind_managed(vec![binding], config(RestartPolicy::OneForOne, 3))
.unwrap();
let launched = Arc::new(Mutex::new(None));
let publication = Arc::clone(&launched);
let (launcher, mut launcher_join) = lab
.state
.create_task(root, Budget::INFINITE, async move {
let cx = Cx::current().unwrap();
*publication.lock() = Some(managed.spawn(&cx).unwrap());
})
.unwrap();
lab.scheduler.lock().schedule(launcher, 0);
lab.run_until_idle();
assert_eq!(launcher_join.try_join(), Ok(Some(())));
let mut handle = launched
.lock()
.take()
.expect("actual managed controller admitted");
assert_eq!(started.lock().len(), 1);
let generation = started.lock()[0].1;
let supervisor_region =
lab.state.region(generation.region).unwrap().parent.unwrap();
assert_ne!(supervisor_region, root);
assert_eq!(
started.lock()[0].2.telemetry_snapshot(0).recv_waiter_count,
1
);
let entered = Arc::new(AtomicUsize::new(0));
let completed = Arc::new(AtomicUsize::new(0));
let finalizer_entered = Arc::clone(&entered);
let finalizer_completed = Arc::clone(&completed);
let (release, mut finalizer_gate) = oneshot::channel::<()>();
assert!(
lab.state
.register_async_finalizer(supervisor_region, async move {
finalizer_entered.fetch_add(1, Ordering::SeqCst);
finalizer_gate.recv_uninterruptible().await.unwrap();
finalizer_completed.fetch_add(1, Ordering::SeqCst);
})
);
started.lock()[0].2.try_send(()).unwrap();
lab.run_until_idle();
assert_eq!(entered.load(Ordering::SeqCst), 1);
assert_eq!(completed.load(Ordering::SeqCst), 0);
assert!(lab.state.task(generation.task).is_none());
assert!(lab.state.region(generation.region).is_none());
assert!(lab.state.region(supervisor_region).is_some());
assert!(lab.state.task(handle.task_id()).is_some());
let mut poll_cx = std::task::Context::from_waker(std::task::Waker::noop());
assert!(
Box::pin(handle.join())
.as_mut()
.poll(&mut poll_cx)
.is_pending(),
"a completed Temporary child is not a completed supervisor root close"
);
if cancel_before_close {
handle.abort();
lab.run_until_idle();
assert_eq!(completed.load(Ordering::SeqCst), 0);
assert!(lab.state.region(supervisor_region).is_some());
assert!(
Box::pin(handle.join())
.as_mut()
.poll(&mut poll_cx)
.is_pending()
);
}
release.send_blocking(()).unwrap();
lab.run_until_idle();
assert_eq!(completed.load(Ordering::SeqCst), 1);
assert!(lab.state.region(supervisor_region).is_none());
assert!(
lab.state.task(handle.task_id()).is_none(),
"actual controller terminal precedes the late-cancel branch"
);
if !cancel_before_close {
handle.abort();
}
let Poll::Ready(Ok(report)) = Box::pin(handle.join()).as_mut().poll(&mut poll_cx)
else {
panic!("actual terminated controller must retain its completed report");
};
if cancel_before_close {
assert!(report.outcome.is_cancelled(), "{report:?}");
} else {
assert!(
report.outcome.is_ok(),
"late cancellation cannot rewrite a published report: {report:?}"
);
}
assert_eq!(
(
report.started,
report.joined,
report.restart_batches,
report.escalations
),
(1, 1, 0, 0)
);
assert_eq!(report.children.len(), 1);
assert_eq!(report.children[0].generation, generation);
assert!(report.children[0].outcome.is_ok());
assert_eq!(report.children[0].task_outcome, Ok(()));
assert!(report.children[0].region_outcome.is_some());
assert!(report.region_outcome.is_some());
assert!(
matches!(report.cleanup_outcome, Some(Outcome::Ok(()))),
"{report:?}"
);
let trace = lab.state.trace_handle().snapshot();
for task in [generation.task, handle.task_id()] {
assert_eq!(trace.iter().filter(|event| event.kind == crate::trace::TraceEventKind::Complete &&
matches!(event.data, crate::trace::TraceData::Task { task: actual, .. } if actual == task)).count(), 1);
}
clean(&mut lab, root);
}
}
#[test]
fn managed_cancel_all_precedes_cleanup_that_waits_for_sibling_cancellation() {
for restart_policy in [
Some(RestartPolicy::OneForAll),
Some(RestartPolicy::RestForOne),
None,
] {
let policy = restart_policy.unwrap_or(RestartPolicy::OneForOne);
let mut lab = LabRuntime::new(LabConfig::new(0x34_0008).max_steps(8192));
let root = lab.state.create_root_region(Budget::INFINITE);
let starts = Arc::new(AtomicUsize::new(0));
let b_cancelled = Arc::new(AtomicUsize::new(0));
let c_pending = Arc::new(AtomicUsize::new(0));
let c_finished = Arc::new(AtomicUsize::new(0));
let trigger = Arc::new(Mutex::new(None));
let published_trigger = Arc::clone(&trigger);
let a_starts = Arc::clone(&starts);
let a = ManagedChildBinding::new(
"a",
ManagedRestartMode::Transient,
move |cx: Cx, generation: ManagedGeneration| {
a_starts.fetch_add(1, Ordering::SeqCst);
let (sender, mut receiver) = mpsc::channel::<()>(1);
*published_trigger.lock() = Some(sender);
async move {
if generation.number > 1 {
return Outcome::Ok(());
}
match receiver.recv(&cx).await {
Ok(()) => Outcome::Err("restart trigger"),
Err(_) => Outcome::Cancelled(cx.cancel_reason().unwrap()),
}
}
},
);
let (release_b, b_gate) = oneshot::channel::<()>();
let (witness, c_gate) = oneshot::channel::<()>();
let b_state = Arc::new(Mutex::new(Some((b_gate, witness))));
let b_starts = Arc::clone(&starts);
let b_observed = Arc::clone(&b_cancelled);
let b = ManagedChildBinding::new(
"b",
ManagedRestartMode::Transient,
move |cx: Cx, generation: ManagedGeneration| {
b_starts.fetch_add(1, Ordering::SeqCst);
let gates =
(generation.number == 1).then(|| b_state.lock().take().unwrap());
let observed = Arc::clone(&b_observed);
async move {
let Some((mut release, witness)) = gates else {
return Outcome::Ok(());
};
let (_keep_sender, mut receiver) = mpsc::channel::<()>(1);
assert!(receiver.recv(&cx).await.is_err());
assert!(cx.cancel_reason().is_some());
observed.fetch_add(1, Ordering::SeqCst);
release.recv_uninterruptible().await.unwrap();
witness.send_blocking(()).unwrap();
Outcome::<(), &'static str>::Cancelled(cx.cancel_reason().unwrap())
}
},
);
let c_state = Arc::new(Mutex::new(Some(c_gate)));
let c_starts = Arc::clone(&starts);
let c_waited = Arc::clone(&c_pending);
let c_done = Arc::clone(&c_finished);
let c = ManagedChildBinding::new(
"c",
ManagedRestartMode::Transient,
move |cx: Cx, generation: ManagedGeneration| {
c_starts.fetch_add(1, Ordering::SeqCst);
let gate = (generation.number == 1).then(|| c_state.lock().take().unwrap());
let waited = Arc::clone(&c_waited);
let done = Arc::clone(&c_done);
async move {
let Some(mut witness) = gate else {
return Outcome::Ok(());
};
let (_keep_sender, mut receiver) = mpsc::channel::<()>(1);
assert!(receiver.recv(&cx).await.is_err());
assert!(cx.cancel_reason().is_some());
let mut waiting = std::pin::pin!(witness.recv_uninterruptible());
poll_fn(|poll_cx| {
let result = waiting.as_mut().poll(poll_cx);
if result.is_pending() {
waited.fetch_add(1, Ordering::SeqCst);
}
result
})
.await
.unwrap();
done.fetch_add(1, Ordering::SeqCst);
Outcome::<(), &'static str>::Cancelled(cx.cancel_reason().unwrap())
}
},
);
let managed = topology(&["a", "b", "c"], policy)
.bind_managed(vec![a, b, c], config(policy, 1))
.unwrap();
let output = Arc::new(Mutex::new(None));
let published = Arc::clone(&output);
let (parent, mut join) = lab
.state
.create_task(root, Budget::INFINITE, async move {
*published.lock() = Some(managed.run(&Cx::current().unwrap()).await);
})
.unwrap();
lab.scheduler.lock().schedule(parent, 0);
lab.run_until_idle();
assert_eq!(starts.load(Ordering::SeqCst), 3);
assert!(join.try_join().unwrap().is_none());
if restart_policy.is_some() {
trigger.lock().as_ref().unwrap().try_send(()).unwrap();
} else {
join.abort();
}
lab.run_until_idle();
assert!(
c_pending.load(Ordering::SeqCst) > 0,
"first drained child's actual cleanup must park"
);
assert_eq!(
b_cancelled.load(Ordering::SeqCst),
1,
"second sibling must observe cancellation while first cleanup is still Pending"
);
assert_eq!(c_finished.load(Ordering::SeqCst), 0);
assert_eq!(
starts.load(Ordering::SeqCst),
3,
"no generation can replace undrained work"
);
assert!(output.lock().is_none());
assert!(join.try_join().unwrap().is_none());
release_b.send_blocking(()).unwrap();
lab.run_until_idle();
if restart_policy.is_some() {
assert!(matches!(join.try_join(), Ok(Some(()))));
} else {
assert!(matches!(join.try_join(), Err(JoinError::Cancelled(_))));
}
let report = output
.lock()
.take()
.expect("all acknowledged cleanup joins before report");
assert_eq!(c_finished.load(Ordering::SeqCst), 1);
assert_eq!(b_cancelled.load(Ordering::SeqCst), 1);
assert_eq!(report.restart_batches, u64::from(restart_policy.is_some()));
assert_eq!(report.started, if restart_policy.is_some() { 6 } else { 3 });
assert_eq!(report.joined, report.started);
assert_eq!(report.outcome.is_ok(), restart_policy.is_some());
assert_eq!(report.outcome.is_cancelled(), restart_policy.is_none());
if restart_policy.is_none() {
for completed in report.children.iter().filter(|child| child.name != "a") {
assert!(completed.shutdown_requested_before_completion);
assert!(completed.outcome.is_cancelled());
assert!(
completed.task_outcome.is_ok(),
"acknowledged cleanup returns its actual task value"
);
}
}
clean(&mut lab, root);
}
}
#[test]
fn managed_raw_user_return_does_not_hide_actual_unacknowledged_cancellation() {
for returned_error in [false, true] {
let report = run_case(move |cx| async move {
let binding = ManagedChildBinding::new(
"child",
ManagedRestartMode::Transient,
move |child: Cx, _| async move {
child.cancel_with(
crate::types::CancelKind::User,
Some("independent unacknowledged cancellation"),
);
if returned_error {
Outcome::Err("late domain error")
} else {
Outcome::Ok(())
}
},
);
topology(&["child"], RestartPolicy::OneForOne)
.bind_managed(vec![binding], config(RestartPolicy::OneForOne, 3))
.unwrap()
.run(&cx)
.await
});
assert_eq!(report.started, 1);
assert_eq!(report.joined, 1);
assert_eq!(
report.restart_batches, 0,
"transient cancellation cannot be recast as restartable raw Err"
);
let completed = &report.children[0];
assert_eq!(completed.outcome.is_err(), returned_error);
assert_eq!(completed.outcome.is_ok(), !returned_error);
assert!(matches!(
completed.task_outcome,
Err(JoinError::Cancelled(_))
));
assert!(!completed.shutdown_requested_before_completion);
}
}
#[test]
fn managed_transient_completion_during_bounded_scan_is_not_resurrected() {
use std::sync::atomic::AtomicBool;
let mut lab = LabRuntime::new(LabConfig::new(0x34_0009).max_steps(32_768));
let root = lab.state.create_root_region(Budget::INFINITE);
let names: Vec<_> = (0..33).map(|index| format!("child-{index:02}")).collect();
let name_refs: Vec<_> = names.iter().map(String::as_str).collect();
let starts = Arc::new(Mutex::new(Vec::new()));
let (release_first, wait_first) = oneshot::channel::<()>();
let first_gate = Arc::new(Mutex::new(Some(wait_first)));
let mut bindings = Vec::new();
for (index, name) in names.iter().enumerate() {
let log = Arc::clone(&starts);
let gate = Arc::clone(&first_gate);
let mode = if index == 0 || index == 32 {
ManagedRestartMode::Transient
} else {
ManagedRestartMode::Temporary
};
bindings.push(ManagedChildBinding::new(
name.clone(),
mode,
move |child: Cx, generation: ManagedGeneration| {
log.lock().push((index, generation));
let first = (index == 0 && generation.number == 1)
.then(|| gate.lock().take().unwrap());
async move {
if generation.number > 1 {
return Outcome::Ok(());
}
if let Some(mut wait) = first {
wait.recv_uninterruptible().await.unwrap();
return Outcome::Ok(());
}
if index == 32 {
return Outcome::Err(());
}
let (_keep_sender, mut receiver) = mpsc::channel::<()>(1);
assert!(receiver.recv(&child).await.is_err());
Outcome::Cancelled(child.cancel_reason().unwrap())
}
},
));
}
let managed = topology(&name_refs, RestartPolicy::OneForAll)
.bind_managed(bindings, config(RestartPolicy::OneForAll, 1))
.unwrap();
let permit_poll = Arc::new(AtomicBool::new(false));
let permitted = Arc::clone(&permit_poll);
let inner_polls = Arc::new(AtomicUsize::new(0));
let observed_polls = Arc::clone(&inner_polls);
let (parent, mut join) = lab
.state
.create_task(root, Budget::INFINITE, async move {
let cx = Cx::current().unwrap();
let mut execution = Box::pin(managed.run(&cx));
poll_fn(|poll_cx| {
if !permitted.swap(false, Ordering::SeqCst) {
return Poll::Pending;
}
observed_polls.fetch_add(1, Ordering::SeqCst);
execution.as_mut().poll(poll_cx)
})
.await
})
.unwrap();
for _ in 0..256 {
if starts.lock().len() == 33 {
break;
}
permit_poll.store(true, Ordering::SeqCst);
lab.scheduler.lock().schedule(parent, 0);
lab.run_until_idle();
}
assert_eq!(starts.lock().len(), 33);
let first = starts.lock()[0].1;
let trigger = starts.lock()[32].1;
assert!(lab.state.task(first.task).is_some());
assert!(
lab.state.task(trigger.task).is_none(),
"actual final block's failing child already completed"
);
let before = inner_polls.load(Ordering::SeqCst);
permit_poll.store(true, Ordering::SeqCst);
lab.scheduler.lock().schedule(parent, 0);
lab.run_until_idle();
assert_eq!(inner_polls.load(Ordering::SeqCst), before + 1);
assert!(join.try_join().unwrap().is_none());
release_first.send_blocking(()).unwrap();
lab.run_until_idle();
assert_eq!(
inner_polls.load(Ordering::SeqCst),
before + 1,
"normal completion occurs while the controller scan remains suspended"
);
assert!(lab.state.task(first.task).is_none());
assert!(
lab.state
.trace_handle()
.snapshot()
.iter()
.any(|event| event.kind == crate::trace::TraceEventKind::Complete
&& matches!(event.data, crate::trace::TraceData::Task { task, region }
if task == first.task && region == first.region)),
"full canonical task/region completion is the causal witness"
);
let mut result = None;
for _ in 0..512 {
permit_poll.store(true, Ordering::SeqCst);
lab.scheduler.lock().schedule(parent, 0);
lab.run_until_idle();
if let Some(report) = join.try_join().unwrap() {
result = Some(report);
break;
}
}
let report = result.expect("bounded actual controller resumes, drains and finishes");
assert_eq!(
report.started, 34,
"only the failed last child gets a replacement"
);
assert_eq!(report.joined, 34);
assert_eq!(report.restart_batches, 1);
assert_eq!(
starts
.lock()
.iter()
.filter(|(index, _)| *index == 0)
.count(),
1
);
assert_eq!(
starts
.lock()
.iter()
.filter(|(index, _)| *index == 32)
.count(),
2
);
let completed = &report.children[0];
assert_eq!(completed.generation, first);
assert!(completed.outcome.is_ok());
assert!(completed.task_outcome.is_ok());
assert!(!completed.shutdown_requested_before_completion);
assert!(report.outcome.is_ok());
clean(&mut lab, root);
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct ChildId {
slot: u32,
generation: u32,
}
impl ChildId {
#[must_use]
pub const fn new(slot: u32, generation: u32) -> Self {
Self { slot, generation }
}
#[must_use]
pub const fn slot(self) -> u32 {
self.slot
}
#[must_use]
pub const fn generation(self) -> u32 {
self.generation
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ChildHandle {
id: ChildId,
name: ChildName,
task_id: TaskId,
}
impl ChildHandle {
#[must_use]
pub fn new(id: ChildId, name: ChildName, task_id: TaskId) -> Self {
Self { id, name, task_id }
}
#[must_use]
pub const fn id(&self) -> ChildId {
self.id
}
#[must_use]
pub fn name(&self) -> &ChildName {
&self.name
}
#[must_use]
pub const fn task_id(&self) -> TaskId {
self.task_id
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DynamicChildRecord {
handle: ChildHandle,
restart: SupervisionStrategy,
shutdown_budget: Budget,
start_sequence: u64,
}
impl DynamicChildRecord {
#[must_use]
pub fn new(
handle: ChildHandle,
restart: SupervisionStrategy,
shutdown_budget: Budget,
start_sequence: u64,
) -> Self {
Self {
handle,
restart,
shutdown_budget,
start_sequence,
}
}
#[must_use]
pub const fn handle(&self) -> &ChildHandle {
&self.handle
}
#[must_use]
pub const fn id(&self) -> ChildId {
self.handle.id()
}
#[must_use]
pub fn name(&self) -> &ChildName {
self.handle.name()
}
#[must_use]
pub const fn task_id(&self) -> TaskId {
self.handle.task_id()
}
#[must_use]
pub const fn restart(&self) -> &SupervisionStrategy {
&self.restart
}
#[must_use]
pub const fn shutdown_budget(&self) -> Budget {
self.shutdown_budget
}
#[must_use]
pub const fn start_sequence(&self) -> u64 {
self.start_sequence
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum DynamicChildError {
DuplicateChildName(ChildName),
ChildIdExhausted,
StartSequenceExhausted,
}
impl std::fmt::Display for DynamicChildError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::DuplicateChildName(name) => {
write!(f, "duplicate dynamic child name: {name}")
}
Self::ChildIdExhausted => write!(f, "dynamic child id space exhausted"),
Self::StartSequenceExhausted => {
write!(f, "dynamic child start sequence exhausted")
}
}
}
}
impl std::error::Error for DynamicChildError {}
#[derive(Debug, Default)]
pub struct DynamicChildTable {
entries: Vec<Option<DynamicChildRecord>>,
generations: Vec<u32>,
free_slots: Vec<usize>,
by_name: BTreeMap<ChildName, ChildId>,
next_start_sequence: u64,
}
impl DynamicChildTable {
#[must_use]
pub fn new() -> Self {
Self::default()
}
#[must_use]
pub fn len(&self) -> usize {
self.by_name.len()
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.by_name.is_empty()
}
pub fn insert_started(
&mut self,
name: impl Into<ChildName>,
task_id: TaskId,
restart: SupervisionStrategy,
shutdown_budget: Budget,
) -> Result<ChildHandle, DynamicChildError> {
let name = name.into();
if self.by_name.contains_key(name.as_str()) {
return Err(DynamicChildError::DuplicateChildName(name));
}
let start_sequence = self.next_start_sequence;
let next_start_sequence = self
.next_start_sequence
.checked_add(1)
.ok_or(DynamicChildError::StartSequenceExhausted)?;
let (slot, generation) = self.allocate_id_parts()?;
let id = ChildId::new(slot, generation);
let handle = ChildHandle::new(id, name.clone(), task_id);
let record =
DynamicChildRecord::new(handle.clone(), restart, shutdown_budget, start_sequence);
let slot_index = usize::try_from(slot).map_err(|_| DynamicChildError::ChildIdExhausted)?;
self.entries[slot_index] = Some(record);
self.by_name.insert(name, id);
self.next_start_sequence = next_start_sequence;
Ok(handle)
}
pub fn remove(&mut self, id: ChildId) -> Option<DynamicChildRecord> {
let slot = usize::try_from(id.slot()).ok()?;
if self.generations.get(slot).copied()? != id.generation() {
return None;
}
let record = self.entries.get_mut(slot)?.take()?;
self.by_name.remove(record.name().as_str());
self.free_slots.push(slot);
Some(record)
}
pub fn remove_by_name(&mut self, name: &str) -> Option<DynamicChildRecord> {
let id = *self.by_name.get(name)?;
self.remove(id)
}
#[must_use]
pub fn get(&self, id: ChildId) -> Option<&DynamicChildRecord> {
let slot = usize::try_from(id.slot()).ok()?;
if self.generations.get(slot).copied()? != id.generation() {
return None;
}
self.entries.get(slot)?.as_ref()
}
#[must_use]
pub fn get_by_name(&self, name: &str) -> Option<&DynamicChildRecord> {
let id = *self.by_name.get(name)?;
self.get(id)
}
#[must_use]
pub fn contains_name(&self, name: &str) -> bool {
self.by_name.contains_key(name)
}
#[must_use]
pub fn which_children(&self) -> Vec<&DynamicChildRecord> {
let mut children = self
.entries
.iter()
.filter_map(std::option::Option::as_ref)
.collect::<Vec<_>>();
children.sort_by_key(|child| child.start_sequence());
children
}
fn allocate_id_parts(&mut self) -> Result<(u32, u32), DynamicChildError> {
if let Some(&slot) = self.free_slots.last() {
let next_generation = self.generations[slot]
.checked_add(1)
.ok_or(DynamicChildError::ChildIdExhausted)?;
self.free_slots.pop();
self.generations[slot] = next_generation;
let slot = u32::try_from(slot).map_err(|_| DynamicChildError::ChildIdExhausted)?;
return Ok((slot, next_generation));
}
let slot_index = self.entries.len();
let slot = u32::try_from(slot_index).map_err(|_| DynamicChildError::ChildIdExhausted)?;
self.entries.push(None);
self.generations.push(0);
Ok((slot, 0))
}
}
impl BackoffStrategy {
#[must_use]
pub fn delay_for_attempt(&self, attempt: u32) -> Option<Duration> {
match self {
Self::None => None,
Self::Fixed(d) => Some(*d),
Self::Exponential {
initial,
max,
multiplier,
} => {
let safe_multiplier = if multiplier.is_finite() && *multiplier >= 0.0 {
*multiplier
} else {
2.0
};
#[allow(clippy::cast_precision_loss)]
let exp = i32::try_from(attempt).unwrap_or(30).min(30);
let base_secs = initial.as_secs_f64() * safe_multiplier.powi(exp);
let safe_secs = if base_secs.is_finite() && base_secs >= 0.0 {
base_secs
} else {
max.as_secs_f64()
};
let capped_secs = safe_secs.min(max.as_secs_f64());
let delay = Duration::try_from_secs_f64(capped_secs)
.unwrap_or(*max)
.min(*max);
Some(delay)
}
}
}
}
#[derive(Debug, Clone)]
pub struct RestartHistory {
restarts: Vec<u64>, config: RestartConfig,
}
impl RestartHistory {
#[must_use]
pub fn new(config: RestartConfig) -> Self {
Self {
restarts: Vec::new(),
config,
}
}
#[must_use]
pub fn can_restart(&self, now: u64) -> bool {
let window_nanos = duration_nanos_u64(self.config.window);
let cutoff = now.saturating_sub(window_nanos);
let recent_count = self.restarts.iter().filter(|&&t| t >= cutoff).count();
recent_count < self.config.max_restarts as usize
}
pub fn record_restart(&mut self, now: u64) {
let window_nanos = duration_nanos_u64(self.config.window);
let cutoff = now.saturating_sub(window_nanos);
self.restarts.retain(|&t| t >= cutoff);
self.restarts.push(now);
}
pub fn try_record_restart(&mut self, now: u64) -> Option<(u32, Option<Duration>)> {
if !self.can_restart(now) {
return None;
}
let attempt = self.recent_restart_count(now) as u32 + 1;
let delay = self.next_delay(now);
self.record_restart(now);
Some((attempt, delay))
}
#[must_use]
pub fn recent_restart_count(&self, now: u64) -> usize {
let window_nanos = duration_nanos_u64(self.config.window);
let cutoff = now.saturating_sub(window_nanos);
self.restarts.iter().filter(|&&t| t >= cutoff).count()
}
#[must_use]
pub fn next_delay(&self, now: u64) -> Option<Duration> {
let attempt = self.recent_restart_count(now) as u32;
self.config.backoff.delay_for_attempt(attempt)
}
#[must_use]
pub fn config(&self) -> &RestartConfig {
&self.config
}
pub fn can_restart_with_budget(&self, now: u64, budget: &Budget) -> Result<(), BudgetRefusal> {
if !self.can_restart(now) {
return Err(BudgetRefusal::WindowExhausted {
max_restarts: self.config.max_restarts,
window: self.config.window,
});
}
if self.config.restart_cost > 0 {
if let Some(remaining) = budget.cost_quota {
if remaining < self.config.restart_cost {
return Err(BudgetRefusal::InsufficientCost {
required: self.config.restart_cost,
remaining,
});
}
}
}
if let Some(min_remaining) = self.config.min_remaining_for_restart {
if let Some(deadline) = budget.deadline {
let now_time = crate::types::id::Time::from_nanos(now);
let remaining = budget.remaining_time(now_time);
match remaining {
None => {
return Err(BudgetRefusal::DeadlineTooClose {
min_required: min_remaining,
remaining: Duration::ZERO,
});
}
Some(rem) if rem < min_remaining => {
return Err(BudgetRefusal::DeadlineTooClose {
min_required: min_remaining,
remaining: rem,
});
}
_ => {} }
let _ = deadline;
}
}
if self.config.min_polls_for_restart > 0
&& budget.poll_quota < self.config.min_polls_for_restart
{
return Err(BudgetRefusal::InsufficientPolls {
min_required: self.config.min_polls_for_restart,
remaining: budget.poll_quota,
});
}
Ok(())
}
#[must_use]
pub fn intensity(&self, now: u64) -> f64 {
let count = self.recent_restart_count(now);
if count == 0 {
return 0.0;
}
let window_secs = self.config.window.as_secs_f64();
if window_secs <= 0.0 {
return 0.0;
}
#[allow(clippy::cast_precision_loss)]
let intensity = count as f64 / window_secs;
intensity
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum BudgetRefusal {
WindowExhausted {
max_restarts: u32,
window: Duration,
},
InsufficientCost {
required: u64,
remaining: u64,
},
DeadlineTooClose {
min_required: Duration,
remaining: Duration,
},
InsufficientPolls {
min_required: u32,
remaining: u32,
},
}
impl std::fmt::Display for BudgetRefusal {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::WindowExhausted {
max_restarts,
window,
} => write!(
f,
"restart window exhausted: {max_restarts} restarts in {window:?}"
),
Self::InsufficientCost {
required,
remaining,
} => write!(
f,
"insufficient cost budget: need {required}, have {remaining}"
),
Self::DeadlineTooClose {
min_required,
remaining,
} => write!(
f,
"deadline too close: need {min_required:?} remaining, have {remaining:?}"
),
Self::InsufficientPolls {
min_required,
remaining,
} => write!(
f,
"insufficient poll budget: need {min_required}, have {remaining}"
),
}
}
}
impl std::error::Error for BudgetRefusal {}
#[derive(Debug, Clone)]
pub struct RestartIntensityWindow {
timestamps: Vec<u64>,
window: Duration,
storm_threshold: f64,
}
impl RestartIntensityWindow {
#[must_use]
pub fn new(window: Duration, storm_threshold: f64) -> Self {
validate_storm_threshold(storm_threshold);
Self {
timestamps: Vec::new(),
window,
storm_threshold,
}
}
pub fn record(&mut self, now: u64) {
let window_nanos = duration_nanos_u64(self.window);
let cutoff = now.saturating_sub(window_nanos);
self.timestamps.retain(|&t| t >= cutoff);
self.timestamps.push(now);
}
#[must_use]
pub fn intensity(&self, now: u64) -> f64 {
let window_nanos = duration_nanos_u64(self.window);
let cutoff = now.saturating_sub(window_nanos);
let count = self.timestamps.iter().filter(|&&t| t >= cutoff).count();
if count == 0 {
return 0.0;
}
let window_secs = self.window.as_secs_f64();
if window_secs <= 0.0 {
return 0.0;
}
#[allow(clippy::cast_precision_loss)]
let intensity = count as f64 / window_secs;
intensity
}
#[must_use]
pub fn is_storm(&self, now: u64) -> bool {
self.intensity(now) > self.storm_threshold
}
#[must_use]
pub fn count(&self, now: u64) -> usize {
let window_nanos = duration_nanos_u64(self.window);
let cutoff = now.saturating_sub(window_nanos);
self.timestamps.iter().filter(|&&t| t >= cutoff).count()
}
#[must_use]
pub fn storm_threshold(&self) -> f64 {
self.storm_threshold
}
#[must_use]
pub fn window(&self) -> Duration {
self.window
}
}
#[derive(Debug, Clone, Copy)]
pub struct StormMonitorConfig {
pub alpha: f64,
pub expected_rate: f64,
pub min_observations: u64,
pub tolerance: f64,
}
impl Default for StormMonitorConfig {
fn default() -> Self {
Self {
alpha: 0.01,
expected_rate: 0.05, min_observations: 3,
tolerance: 1.2,
}
}
}
#[derive(Debug)]
pub struct RestartStormMonitor {
config: StormMonitorConfig,
e_value: f64,
threshold: f64,
observations: u64,
log_e_value: f64,
peak_e_value: f64,
alert_count: u64,
}
impl RestartStormMonitor {
#[must_use]
pub fn new(config: StormMonitorConfig) -> Self {
assert!(
config.alpha > 0.0 && config.alpha < 1.0,
"alpha must be in (0, 1), got {}",
config.alpha
);
assert!(
config.expected_rate > 0.0,
"expected_rate must be > 0, got {}",
config.expected_rate
);
assert!(
config.tolerance >= 1.0,
"tolerance must be >= 1.0, got {}",
config.tolerance
);
let threshold = 1.0 / config.alpha;
Self {
config,
e_value: 1.0,
threshold,
observations: 0,
log_e_value: 0.0,
peak_e_value: 1.0,
alert_count: 0,
}
}
pub fn observe_intensity(&mut self, intensity: f64) -> crate::obligation::eprocess::AlertState {
let was_alert = self.is_alert();
self.observations += 1;
let ratio = intensity / self.config.expected_rate;
let normalizer = self.config.tolerance;
let lr = ratio.max(1.0) / normalizer;
self.log_e_value += lr.ln();
if self.log_e_value < 0.0 {
self.log_e_value = 0.0;
}
self.e_value = self.log_e_value.exp();
if self.e_value > self.peak_e_value {
self.peak_e_value = self.e_value;
}
if !was_alert
&& self.e_value >= self.threshold
&& self.observations >= self.config.min_observations
{
self.alert_count += 1;
}
self.alert_state()
}
pub fn observe_from_window(
&mut self,
window: &RestartIntensityWindow,
now: u64,
) -> crate::obligation::eprocess::AlertState {
self.observe_intensity(window.intensity(now))
}
#[must_use]
pub fn alert_state(&self) -> crate::obligation::eprocess::AlertState {
use crate::obligation::eprocess::AlertState;
if self.observations < self.config.min_observations {
return AlertState::Clear;
}
if self.e_value >= self.threshold {
AlertState::Alert
} else if self.e_value > 1.0 {
AlertState::Watching
} else {
AlertState::Clear
}
}
#[must_use]
pub fn is_alert(&self) -> bool {
self.alert_state() == crate::obligation::eprocess::AlertState::Alert
}
#[must_use]
pub fn e_value(&self) -> f64 {
self.e_value
}
#[must_use]
pub fn threshold(&self) -> f64 {
self.threshold
}
#[must_use]
pub fn observations(&self) -> u64 {
self.observations
}
#[must_use]
pub fn peak_e_value(&self) -> f64 {
self.peak_e_value
}
#[must_use]
pub fn alert_count(&self) -> u64 {
self.alert_count
}
#[must_use]
pub fn config(&self) -> &StormMonitorConfig {
&self.config
}
pub fn reset(&mut self) {
self.e_value = 1.0;
self.log_e_value = 0.0;
self.peak_e_value = 1.0;
self.observations = 0;
self.alert_count = 0;
}
#[must_use]
pub fn snapshot(&self) -> StormMonitorSnapshot {
StormMonitorSnapshot {
e_value: self.e_value,
threshold: self.threshold,
observations: self.observations,
alert_state: self.alert_state(),
peak_e_value: self.peak_e_value,
alert_count: self.alert_count,
}
}
}
#[derive(Debug, Clone)]
pub struct StormMonitorSnapshot {
pub e_value: f64,
pub threshold: f64,
pub observations: u64,
pub alert_state: crate::obligation::eprocess::AlertState,
pub peak_e_value: f64,
pub alert_count: u64,
}
impl std::fmt::Display for StormMonitorSnapshot {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"StormMonitor[{}]: e={:.4} threshold={:.1} obs={} peak={:.4} alerts={}",
self.alert_state,
self.e_value,
self.threshold,
self.observations,
self.peak_e_value,
self.alert_count,
)
}
}
#[derive(Debug, Clone)]
pub struct RestartTrackerConfig {
pub restart: RestartConfig,
pub storm_threshold: Option<f64>,
pub storm_monitor: StormMonitorConfig,
auto_align_storm_expected_rate: bool,
}
impl RestartTrackerConfig {
#[must_use]
pub fn from_restart(restart: RestartConfig) -> Self {
Self {
restart,
storm_threshold: None,
storm_monitor: StormMonitorConfig::default(),
auto_align_storm_expected_rate: true,
}
}
#[must_use]
pub fn with_storm_detection(mut self, threshold: f64) -> Self {
validate_storm_threshold(threshold);
self.storm_threshold = Some(threshold);
self
}
#[must_use]
pub fn with_storm_monitor(mut self, config: StormMonitorConfig) -> Self {
self.storm_monitor = config;
self.auto_align_storm_expected_rate = false;
self
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RestartVerdict {
Allowed {
attempt: u32,
delay: Option<Duration>,
},
Denied {
refusal: BudgetRefusal,
},
}
impl RestartVerdict {
#[must_use]
pub fn is_allowed(&self) -> bool {
matches!(self, Self::Allowed { .. })
}
}
#[derive(Debug)]
pub struct RestartTracker {
history: RestartHistory,
intensity: Option<RestartIntensityWindow>,
storm: Option<RestartStormMonitor>,
}
impl RestartTracker {
#[must_use]
pub fn new(config: RestartTrackerConfig) -> Self {
let window = config.restart.window;
let (intensity, storm) = match config.storm_threshold {
Some(threshold) => (
Some(RestartIntensityWindow::new(window, threshold)),
Some(RestartStormMonitor::new({
let mut storm_monitor = config.storm_monitor;
if config.auto_align_storm_expected_rate {
storm_monitor.expected_rate = threshold / storm_monitor.tolerance;
}
storm_monitor
})),
),
None => (None, None),
};
let history = RestartHistory::new(config.restart);
Self {
history,
intensity,
storm,
}
}
#[must_use]
pub fn from_restart_config(config: RestartConfig) -> Self {
Self::new(RestartTrackerConfig::from_restart(config))
}
#[must_use]
pub fn evaluate(&self, now: u64) -> RestartVerdict {
if !self.history.can_restart(now) {
return RestartVerdict::Denied {
refusal: BudgetRefusal::WindowExhausted {
max_restarts: self.history.config().max_restarts,
window: self.history.config().window,
},
};
}
let attempt = self.history.recent_restart_count(now) as u32 + 1;
let delay = self.history.next_delay(now);
RestartVerdict::Allowed { attempt, delay }
}
#[must_use]
pub fn evaluate_with_budget(&self, now: u64, budget: &Budget) -> RestartVerdict {
if let Err(refusal) = self.history.can_restart_with_budget(now, budget) {
return RestartVerdict::Denied { refusal };
}
let attempt = self.history.recent_restart_count(now) as u32 + 1;
let delay = self.history.next_delay(now);
RestartVerdict::Allowed { attempt, delay }
}
pub fn record(&mut self, now: u64) {
self.history.record_restart(now);
if let Some(ref mut intensity) = self.intensity {
intensity.record(now);
if let Some(ref mut storm) = self.storm {
storm.observe_from_window(intensity, now);
}
}
}
#[must_use]
pub fn recent_count(&self, now: u64) -> usize {
self.history.recent_restart_count(now)
}
#[must_use]
pub fn intensity(&self, now: u64) -> Option<f64> {
self.intensity.as_ref().map(|w| w.intensity(now))
}
#[must_use]
pub fn is_storm(&self) -> bool {
self.storm
.as_ref()
.is_some_and(RestartStormMonitor::is_alert)
}
#[must_use]
pub fn is_intensity_storm(&self, now: u64) -> bool {
self.intensity.as_ref().is_some_and(|w| w.is_storm(now))
}
#[must_use]
pub fn history(&self) -> &RestartHistory {
&self.history
}
#[must_use]
pub fn storm_snapshot(&self) -> Option<StormMonitorSnapshot> {
self.storm.as_ref().map(RestartStormMonitor::snapshot)
}
pub fn reset(&mut self) {
self.history = RestartHistory::new(self.history.config().clone());
if let Some(ref mut intensity) = self.intensity {
*intensity =
RestartIntensityWindow::new(intensity.window(), intensity.storm_threshold());
}
if let Some(ref mut storm) = self.storm {
storm.reset();
}
}
}
fn validate_storm_threshold(threshold: f64) {
assert!(
threshold.is_finite() && threshold > 0.0,
"storm threshold must be finite and > 0, got {threshold}"
);
}
#[derive(Debug, Clone)]
pub enum SupervisionDecision {
Restart {
task_id: TaskId,
region_id: RegionId,
attempt: u32,
delay: Option<Duration>,
},
Stop {
task_id: TaskId,
region_id: RegionId,
reason: StopReason,
},
Escalate {
task_id: TaskId,
region_id: RegionId,
parent_region_id: Option<RegionId>,
outcome: Outcome<(), ()>,
},
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum StopReason {
ExplicitStop,
RestartBudgetExhausted {
total_restarts: u32,
window: Duration,
},
BudgetRefused(BudgetRefusal),
Cancelled(CancelReason),
Panicked,
RegionClosing,
}
#[derive(Debug, Clone)]
pub enum SupervisionEvent {
ActorFailed {
task_id: TaskId,
region_id: RegionId,
outcome: Outcome<(), ()>,
},
DecisionMade {
task_id: TaskId,
region_id: RegionId,
decision: SupervisionDecision,
},
RestartBeginning {
task_id: TaskId,
region_id: RegionId,
attempt: u32,
},
RestartComplete {
task_id: TaskId,
region_id: RegionId,
attempt: u32,
},
RestartFailed {
task_id: TaskId,
region_id: RegionId,
attempt: u32,
outcome: Outcome<(), ()>,
},
BudgetExhausted {
task_id: TaskId,
region_id: RegionId,
total_restarts: u32,
window: Duration,
},
Escalating {
task_id: TaskId,
from_region: RegionId,
to_region: Option<RegionId>,
},
BudgetRefusedRestart {
task_id: TaskId,
region_id: RegionId,
refusal: BudgetRefusal,
},
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum BindingConstraint {
MonotoneSeverity {
outcome_kind: &'static str,
},
ExplicitStopStrategy,
EscalateStrategy,
RestartAllowed {
attempt: u32,
},
WindowExhausted {
max_restarts: u32,
window: Duration,
},
InsufficientCost {
required: u64,
remaining: u64,
},
DeadlineTooClose {
min_required: Duration,
remaining: Duration,
},
InsufficientPolls {
min_required: u32,
remaining: u32,
},
}
impl std::fmt::Display for BindingConstraint {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::MonotoneSeverity { outcome_kind } => {
write!(f, "monotone severity: {outcome_kind} is not restartable")
}
Self::ExplicitStopStrategy => write!(f, "strategy is Stop"),
Self::EscalateStrategy => write!(f, "strategy is Escalate"),
Self::RestartAllowed { attempt } => {
write!(f, "restart allowed (attempt {attempt})")
}
Self::WindowExhausted {
max_restarts,
window,
} => write!(f, "window exhausted: {max_restarts} restarts in {window:?}"),
Self::InsufficientCost {
required,
remaining,
} => write!(f, "insufficient cost: need {required}, have {remaining}"),
Self::DeadlineTooClose {
min_required,
remaining,
} => write!(
f,
"deadline too close: need {min_required:?}, have {remaining:?}"
),
Self::InsufficientPolls {
min_required,
remaining,
} => write!(
f,
"insufficient polls: need {min_required}, have {remaining}"
),
}
}
}
#[derive(Debug, Clone)]
pub struct EvidenceEntry {
pub timestamp: u64,
pub task_id: TaskId,
pub region_id: RegionId,
pub outcome: Outcome<(), ()>,
pub strategy_kind: &'static str,
pub decision: SupervisionDecision,
pub binding_constraint: BindingConstraint,
}
impl EvidenceEntry {
#[must_use]
pub fn to_evidence_record(&self) -> crate::evidence::EvidenceRecord {
use crate::evidence::{
EvidenceDetail, EvidenceRecord, Subsystem, SupervisionDetail, Verdict,
};
let (verdict, detail) = match &self.binding_constraint {
BindingConstraint::MonotoneSeverity { outcome_kind } => (
Verdict::Stop,
SupervisionDetail::MonotoneSeverity {
outcome_kind: outcome_kind.to_string(),
},
),
BindingConstraint::ExplicitStopStrategy => {
(Verdict::Stop, SupervisionDetail::ExplicitStop)
}
BindingConstraint::EscalateStrategy => {
(Verdict::Escalate, SupervisionDetail::ExplicitEscalate)
}
BindingConstraint::RestartAllowed { attempt } => {
let delay = match &self.decision {
SupervisionDecision::Restart { delay, .. } => *delay,
_ => None,
};
(
Verdict::Restart,
SupervisionDetail::RestartAllowed {
attempt: *attempt,
delay,
},
)
}
BindingConstraint::WindowExhausted {
max_restarts,
window,
} => (
Verdict::Stop,
SupervisionDetail::WindowExhausted {
max_restarts: *max_restarts,
window: *window,
},
),
BindingConstraint::InsufficientCost {
required,
remaining,
} => (
Verdict::Stop,
SupervisionDetail::BudgetRefused {
constraint: format!("insufficient cost: need {required}, have {remaining}"),
},
),
BindingConstraint::DeadlineTooClose {
min_required,
remaining,
} => (
Verdict::Stop,
SupervisionDetail::BudgetRefused {
constraint: format!(
"deadline too close: need {min_required:?}, have {remaining:?}"
),
},
),
BindingConstraint::InsufficientPolls {
min_required,
remaining,
} => (
Verdict::Stop,
SupervisionDetail::BudgetRefused {
constraint: format!(
"insufficient polls: need {min_required}, have {remaining}"
),
},
),
};
EvidenceRecord {
timestamp: self.timestamp,
task_id: self.task_id,
region_id: self.region_id,
subsystem: Subsystem::Supervision,
verdict,
detail: EvidenceDetail::Supervision(detail),
}
}
}
#[derive(Debug, Clone, Default)]
pub struct EvidenceLedger {
entries: Vec<EvidenceEntry>,
}
impl EvidenceLedger {
#[must_use]
pub fn new() -> Self {
Self {
entries: Vec::new(),
}
}
pub fn push(&mut self, entry: EvidenceEntry) {
self.entries.push(entry);
}
#[must_use]
pub fn entries(&self) -> &[EvidenceEntry] {
&self.entries
}
#[must_use]
pub fn len(&self) -> usize {
self.entries.len()
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.entries.is_empty()
}
pub fn for_task(&self, task_id: TaskId) -> impl Iterator<Item = &EvidenceEntry> {
self.entries.iter().filter(move |e| e.task_id == task_id)
}
pub fn with_constraint<F>(&self, predicate: F) -> impl Iterator<Item = &EvidenceEntry>
where
F: Fn(&BindingConstraint) -> bool,
{
self.entries
.iter()
.filter(move |e| predicate(&e.binding_constraint))
}
pub fn clear(&mut self) {
self.entries.clear();
}
}
#[derive(Debug)]
pub struct Supervisor {
strategy: SupervisionStrategy,
history: Option<RestartHistory>,
evidence: EvidenceLedger,
generalized_evidence: crate::evidence::GeneralizedLedger,
}
impl Supervisor {
#[must_use]
pub fn new(strategy: SupervisionStrategy) -> Self {
let history = match &strategy {
SupervisionStrategy::Restart(config) => Some(RestartHistory::new(config.clone())),
_ => None,
};
Self {
strategy,
history,
evidence: EvidenceLedger::new(),
generalized_evidence: crate::evidence::GeneralizedLedger::new(),
}
}
#[must_use]
pub fn strategy(&self) -> &SupervisionStrategy {
&self.strategy
}
fn record_evidence(&mut self, entry: EvidenceEntry) {
let generalized_record = entry.to_evidence_record();
self.evidence.push(entry);
self.generalized_evidence.push(generalized_record);
}
#[allow(clippy::too_many_lines)]
fn decide_err_with_budget(
&mut self,
task_id: TaskId,
region_id: RegionId,
parent_region_id: Option<RegionId>,
now: u64,
budget: Option<&mut Budget>,
) -> (SupervisionDecision, BindingConstraint) {
match &mut self.strategy {
SupervisionStrategy::Stop => (
SupervisionDecision::Stop {
task_id,
region_id,
reason: StopReason::ExplicitStop,
},
BindingConstraint::ExplicitStopStrategy,
),
SupervisionStrategy::Restart(config) => {
let history = self.history.as_mut().expect("history exists for Restart");
if let Some(b) = budget {
if let Err(refusal) = history.can_restart_with_budget(now, b) {
let constraint = match &refusal {
BudgetRefusal::WindowExhausted {
max_restarts,
window,
} => BindingConstraint::WindowExhausted {
max_restarts: *max_restarts,
window: *window,
},
BudgetRefusal::InsufficientCost {
required,
remaining,
} => BindingConstraint::InsufficientCost {
required: *required,
remaining: *remaining,
},
BudgetRefusal::DeadlineTooClose {
min_required,
remaining,
} => BindingConstraint::DeadlineTooClose {
min_required: *min_required,
remaining: *remaining,
},
BudgetRefusal::InsufficientPolls {
min_required,
remaining,
} => BindingConstraint::InsufficientPolls {
min_required: *min_required,
remaining: *remaining,
},
};
let decision = match refusal {
BudgetRefusal::WindowExhausted { .. } => SupervisionDecision::Stop {
task_id,
region_id,
reason: StopReason::RestartBudgetExhausted {
total_restarts: u32::try_from(
history.recent_restart_count(now),
)
.unwrap_or(u32::MAX),
window: config.window,
},
},
_ => SupervisionDecision::Stop {
task_id,
region_id,
reason: StopReason::BudgetRefused(refusal),
},
};
return (decision, constraint);
}
if config.restart_cost > 0 {
b.consume_cost(config.restart_cost);
}
} else if !history.can_restart(now) {
return (
SupervisionDecision::Stop {
task_id,
region_id,
reason: StopReason::RestartBudgetExhausted {
total_restarts: u32::try_from(history.recent_restart_count(now))
.unwrap_or(u32::MAX),
window: config.window,
},
},
BindingConstraint::WindowExhausted {
max_restarts: config.max_restarts,
window: config.window,
},
);
}
let (attempt, delay) = match history.try_record_restart(now) {
Some((attempt, delay)) => (attempt, delay),
None => {
return (
SupervisionDecision::Stop {
task_id,
region_id,
reason: StopReason::RestartBudgetExhausted {
total_restarts: u32::try_from(
history.recent_restart_count(now),
)
.unwrap_or(u32::MAX),
window: config.window,
},
},
BindingConstraint::WindowExhausted {
max_restarts: config.max_restarts,
window: config.window,
},
);
}
};
(
SupervisionDecision::Restart {
task_id,
region_id,
attempt,
delay,
},
BindingConstraint::RestartAllowed { attempt },
)
}
SupervisionStrategy::Escalate => (
SupervisionDecision::Escalate {
task_id,
region_id,
parent_region_id,
outcome: Outcome::Err(()),
},
BindingConstraint::EscalateStrategy,
),
}
}
pub fn on_failure(
&mut self,
task_id: TaskId,
region_id: RegionId,
parent_region_id: Option<RegionId>,
outcome: &Outcome<(), ()>,
now: u64,
) -> SupervisionDecision {
self.on_failure_with_budget(task_id, region_id, parent_region_id, outcome, now, None)
}
pub fn on_failure_with_budget(
&mut self,
task_id: TaskId,
region_id: RegionId,
parent_region_id: Option<RegionId>,
outcome: &Outcome<(), ()>,
now: u64,
budget: Option<&mut Budget>,
) -> SupervisionDecision {
let strategy_kind = match &self.strategy {
SupervisionStrategy::Stop => "Stop",
SupervisionStrategy::Restart(_) => "Restart",
SupervisionStrategy::Escalate => "Escalate",
};
let (decision, constraint) = match outcome {
Outcome::Ok(()) => (
SupervisionDecision::Stop {
task_id,
region_id,
reason: StopReason::ExplicitStop,
},
BindingConstraint::MonotoneSeverity { outcome_kind: "Ok" },
),
Outcome::Cancelled(reason) => (
SupervisionDecision::Stop {
task_id,
region_id,
reason: StopReason::Cancelled(reason.clone()),
},
BindingConstraint::MonotoneSeverity {
outcome_kind: "Cancelled",
},
),
Outcome::Panicked(_) => (
SupervisionDecision::Stop {
task_id,
region_id,
reason: StopReason::Panicked,
},
BindingConstraint::MonotoneSeverity {
outcome_kind: "Panicked",
},
),
Outcome::Err(()) => {
self.decide_err_with_budget(task_id, region_id, parent_region_id, now, budget)
}
};
self.record_evidence(EvidenceEntry {
timestamp: now,
task_id,
region_id,
outcome: outcome.clone(),
strategy_kind,
decision: decision.clone(),
binding_constraint: constraint,
});
decision
}
#[must_use]
pub fn history(&self) -> Option<&RestartHistory> {
self.history.as_ref()
}
#[must_use]
pub fn evidence(&self) -> &EvidenceLedger {
&self.evidence
}
pub fn take_evidence(&mut self) -> EvidenceLedger {
std::mem::take(&mut self.evidence)
}
#[must_use]
pub fn generalized_evidence(&self) -> &crate::evidence::GeneralizedLedger {
&self.generalized_evidence
}
pub fn take_generalized_evidence(&mut self) -> crate::evidence::GeneralizedLedger {
std::mem::take(&mut self.generalized_evidence)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct MonitorRef(u64);
impl MonitorRef {
#[doc(hidden)]
#[must_use]
pub const fn new_for_test(id: u64) -> Self {
Self(id)
}
#[must_use]
pub const fn as_u64(self) -> u64 {
self.0
}
}
impl std::fmt::Display for MonitorRef {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "Mon{}", self.0)
}
}
#[derive(Debug, Clone)]
pub struct Down {
pub monitored: TaskId,
pub reason: Outcome<(), ()>,
pub monitor_ref: MonitorRef,
pub completion_vt: Time,
}
impl Down {
#[must_use]
pub fn sort_key(&self) -> (Time, TaskId) {
(self.completion_vt, self.monitored)
}
}
impl PartialEq for Down {
fn eq(&self, other: &Self) -> bool {
self.monitored == other.monitored
&& self.monitor_ref == other.monitor_ref
&& self.completion_vt == other.completion_vt
}
}
impl Eq for Down {}
#[derive(Debug, Clone)]
struct MonitorEntry {
watcher: TaskId,
watcher_region: RegionId,
monitored: TaskId,
}
#[derive(Debug)]
pub struct MonitorTable {
next_ref: u64,
monitors: BTreeMap<MonitorRef, MonitorEntry>,
by_monitored: BTreeMap<TaskId, Vec<MonitorRef>>,
by_region: BTreeMap<RegionId, Vec<MonitorRef>>,
}
impl Default for MonitorTable {
fn default() -> Self {
Self::new()
}
}
impl MonitorTable {
#[must_use]
pub fn new() -> Self {
Self {
next_ref: 0,
monitors: BTreeMap::new(),
by_monitored: BTreeMap::new(),
by_region: BTreeMap::new(),
}
}
pub fn monitor(
&mut self,
watcher: TaskId,
watcher_region: RegionId,
monitored: TaskId,
) -> MonitorRef {
let mref = MonitorRef(self.next_ref);
self.next_ref += 1;
let entry = MonitorEntry {
watcher,
watcher_region,
monitored,
};
self.monitors.insert(mref, entry);
let refs = self.by_monitored.entry(monitored).or_default();
let pos = refs.binary_search(&mref).unwrap_or_else(|p| p);
refs.insert(pos, mref);
let region_refs = self.by_region.entry(watcher_region).or_default();
let pos = region_refs.binary_search(&mref).unwrap_or_else(|p| p);
region_refs.insert(pos, mref);
mref
}
pub fn demonitor(&mut self, mref: MonitorRef) -> bool {
let Some(entry) = self.monitors.remove(&mref) else {
return false;
};
Self::remove_from_index(&mut self.by_monitored, entry.monitored, mref);
Self::remove_from_index(&mut self.by_region, entry.watcher_region, mref);
true
}
pub fn notify_down(
&mut self,
task: TaskId,
reason: &Outcome<(), ()>,
completion_vt: Time,
) -> Vec<Down> {
let refs = self.by_monitored.remove(&task).unwrap_or_default();
let mut downs = Vec::with_capacity(refs.len());
for mref in refs {
if let Some(entry) = self.monitors.remove(&mref) {
Self::remove_from_index(&mut self.by_region, entry.watcher_region, mref);
downs.push(Down {
monitored: task,
reason: reason.clone(),
monitor_ref: mref,
completion_vt,
});
}
}
downs.sort_by_key(Down::sort_key);
downs
}
pub fn notify_down_batch(
&mut self,
terminations: &[(TaskId, Outcome<(), ()>, Time)],
) -> Vec<Down> {
let mut all_downs = Vec::new();
for (task, reason, vt) in terminations {
all_downs.extend(self.notify_down(*task, reason, *vt));
}
all_downs.sort_by_key(Down::sort_key);
all_downs
}
pub fn cleanup_region(&mut self, region: RegionId) -> usize {
let refs = self.by_region.remove(®ion).unwrap_or_default();
let count = refs.len();
for mref in refs {
if let Some(entry) = self.monitors.remove(&mref) {
Self::remove_from_index(&mut self.by_monitored, entry.monitored, mref);
}
}
count
}
#[must_use]
pub fn len(&self) -> usize {
self.monitors.len()
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.monitors.is_empty()
}
#[must_use]
pub fn watchers_of(&self, task: TaskId) -> &[MonitorRef] {
self.by_monitored.get(&task).map_or(&[], Vec::as_slice)
}
#[must_use]
pub fn watcher_for(&self, mref: MonitorRef) -> Option<TaskId> {
self.monitors.get(&mref).map(|e| e.watcher)
}
#[must_use]
pub fn monitored_for(&self, mref: MonitorRef) -> Option<TaskId> {
self.monitors.get(&mref).map(|e| e.monitored)
}
fn remove_from_index<K>(index: &mut BTreeMap<K, Vec<MonitorRef>>, key: K, mref: MonitorRef)
where
K: Ord + Copy,
{
let remove_bucket = if let Some(bucket) = index.get_mut(&key) {
if let Ok(pos) = bucket.binary_search(&mref) {
bucket.remove(pos);
}
bucket.is_empty()
} else {
false
};
if remove_bucket {
index.remove(&key);
}
}
}
#[derive(Debug, Clone)]
pub enum MonitorEvent {
Established {
watcher: TaskId,
monitored: TaskId,
monitor_ref: MonitorRef,
},
Demonitored {
monitor_ref: MonitorRef,
},
DownProduced {
monitored: TaskId,
watcher: TaskId,
monitor_ref: MonitorRef,
completion_vt: Time,
},
RegionCleanup {
region: RegionId,
count: usize,
},
}
#[cfg(test)]
include!("supervision_tests.rs");
#[cfg(test)]
#[path = "supervision_conformance_tests.rs"]
mod supervision_conformance_tests;
#[cfg(test)]
mod conformance_integration {
use super::supervision_conformance_tests::SupervisionConformanceHarness;
#[test]
fn supervision_conformance_suite() {
crate::test_utils::init_test_logging();
let harness = SupervisionConformanceHarness::new();
let report = harness.run_all_tests();
let mut failures = Vec::new();
let mut passes = 0;
for result in report.results {
if result.passed {
passes += 1;
} else {
let reason = result
.error_message
.unwrap_or_else(|| "no failure reason reported".to_string());
failures.push(format!("{}: {}", result.name, reason));
}
}
assert!(
failures.is_empty(),
"Supervision conformance failures:\n{}",
failures.join("\n")
);
assert!(
passes > 0,
"No conformance tests passed - harness may be broken"
);
crate::test_complete!("supervision_conformance_suite");
}
}