use crate::concurrency::cancel::CancellationToken;
use crate::failure::cause::Cause;
use crate::kernel::{Effect, box_future};
use crate::resource::scope::Scope;
use crate::runtime::{Never, Runtime, run_fork};
use crate::scheduling::clock::Clock;
use crate::scheduling::schedule::{Schedule, ScheduleInput};
use crate::{FiberHandle, FiberId, succeed};
#[derive(Clone)]
pub enum SupervisorPolicy<A: Clone = ()> {
Terminate,
Restart {
schedule: Schedule,
},
RestartWithLimit {
limit: u64,
schedule: Schedule,
},
Escalate,
Ignore {
recover: A,
},
}
#[derive(Clone)]
pub struct Supervisor {
scope: Scope,
token: CancellationToken,
}
impl Supervisor {
pub fn attach(parent: &Scope) -> Self {
let scope = parent.fork();
let token = CancellationToken::new();
let tok = token.clone();
let _ = scope.add_finalizer(Box::new(move |_exit| {
tok.cancel();
succeed::<(), Never, ()>(())
}));
Self { scope, token }
}
pub fn detached() -> Self {
let scope = Scope::make();
let token = CancellationToken::new();
let tok = token.clone();
let _ = scope.add_finalizer(Box::new(move |_exit| {
tok.cancel();
succeed::<(), Never, ()>(())
}));
Self { scope, token }
}
#[inline]
pub fn from_parts(scope: Scope, token: CancellationToken) -> Self {
Self { scope, token }
}
#[inline]
pub fn scope(&self) -> &Scope {
&self.scope
}
#[inline]
pub fn token(&self) -> &CancellationToken {
&self.token
}
#[inline]
pub fn spawn<RT, A, E, R, F, C>(
&self,
runtime: &RT,
policy: SupervisorPolicy<A>,
clock: C,
env: R,
make: F,
) -> FiberHandle<A, Cause<E>>
where
RT: Runtime,
A: Clone + Send + Sync + 'static,
E: Clone + Send + Sync + 'static,
R: Send + 'static,
F: FnMut() -> Effect<A, E, R> + Send + 'static,
C: Clock + Clone + Send + Sync + 'static,
{
let sup = self.clone();
run_fork(runtime, move || {
(supervised(&sup, policy, clock, make), env)
})
}
}
pub fn supervised<A, E, R, F, C>(
supervisor: &Supervisor,
policy: SupervisorPolicy<A>,
clock: C,
mut make: F,
) -> Effect<A, Cause<E>, R>
where
A: Clone + Send + Sync + 'static,
E: Clone + Send + Sync + 'static,
R: 'static,
F: FnMut() -> Effect<A, E, R> + 'static,
C: Clock + Clone + 'static,
{
let token = supervisor.token().clone();
let supervisor_id = FiberId::fresh();
Effect::new_async(move |r: &mut R| {
let clock = clock.clone();
box_future(async move {
let mut schedule_restart: Option<Schedule> = None;
let mut schedule_limited: Option<Schedule> = None;
let mut aggregated: Option<Cause<E>> = None;
let mut restarts_used: u64 = 0;
let mut attempt: u64 = 0;
loop {
if token.is_cancelled() {
return Err(Cause::Interrupt(supervisor_id));
}
match make().run(r).await {
Ok(a) => return Ok(a),
Err(e) => {
let cause = Cause::Fail(e);
match &policy {
SupervisorPolicy::Terminate | SupervisorPolicy::Escalate => {
return Err(cause);
}
SupervisorPolicy::Ignore { recover } => {
return Ok(recover.clone());
}
SupervisorPolicy::Restart { schedule } => {
let sched = schedule_restart.get_or_insert_with(|| schedule.clone());
if let Some(sleep_eff) = sched.next_sleep(&clock, ScheduleInput { attempt }) {
sleep_eff.run(&mut ()).await.unwrap();
attempt = attempt.saturating_add(1);
continue;
} else {
return Err(cause);
}
}
SupervisorPolicy::RestartWithLimit { limit, schedule } => {
if restarts_used >= *limit {
return Err(match aggregated {
None => cause,
Some(prev) => Cause::then(prev, cause),
});
}
aggregated = Some(match aggregated {
None => cause.clone(),
Some(prev) => Cause::then(prev, cause.clone()),
});
let sched = schedule_limited.get_or_insert_with(|| schedule.clone());
if let Some(sleep_eff) = sched.next_sleep(&clock, ScheduleInput { attempt }) {
sleep_eff.run(&mut ()).await.unwrap();
restarts_used = restarts_used.saturating_add(1);
attempt = attempt.saturating_add(1);
continue;
} else {
return Err(aggregated.take().unwrap_or(cause));
}
}
}
}
}
}
})
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::runtime::run_async;
use crate::scheduling::duration::duration as duration_const;
use crate::{TestClock, fail, succeed};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Instant;
mod supervisor_attach {
use super::*;
#[tokio::test]
async fn closing_parent_scope_cancels_supervisor_token_via_child_finalizer() {
let parent = Scope::make();
let sup = Supervisor::attach(&parent);
assert!(!sup.token().is_cancelled());
assert!(parent.close());
assert!(sup.token().is_cancelled());
}
#[test]
fn supervised_loop_observes_interrupt_when_token_cancelled() {
let sup = Supervisor::detached();
let token = sup.token().clone();
let sup_thread = sup.clone();
let calls = Arc::new(AtomicUsize::new(0));
let calls_c = Arc::clone(&calls);
let worker = std::thread::spawn(move || {
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("current_thread runtime");
let eff = supervised(
&sup_thread,
SupervisorPolicy::Restart {
schedule: Schedule::spaced(duration_const::ZERO),
},
TestClock::new(Instant::now()),
move || {
calls_c.fetch_add(1, Ordering::SeqCst);
fail::<(), &'static str, ()>("again")
},
);
rt.block_on(run_async(eff, ()))
});
std::thread::sleep(std::time::Duration::from_millis(30));
token.cancel();
let out = worker
.join()
.expect("join supervised task")
.expect_err("interrupt");
assert!(matches!(out, Cause::Interrupt(_)));
assert!(calls.load(Ordering::SeqCst) >= 1);
}
}
mod supervised_policy {
use super::*;
mod terminate {
use super::*;
#[tokio::test]
async fn returns_ok_on_first_success() {
let sup = Supervisor::detached();
let out = run_async(
supervised(
&sup,
SupervisorPolicy::Terminate,
TestClock::new(Instant::now()),
|| succeed::<u8, &'static str, ()>(7),
),
(),
)
.await
.expect("supervised");
assert_eq!(out, 7);
}
#[tokio::test]
async fn returns_fail_cause_on_first_error_without_retry() {
let sup = Supervisor::detached();
let out = run_async(
supervised(
&sup,
SupervisorPolicy::Terminate,
TestClock::new(Instant::now()),
|| fail::<u8, &'static str, ()>("boom"),
),
(),
)
.await;
assert_eq!(out, Err(Cause::Fail("boom")));
}
}
mod restart {
use super::*;
#[tokio::test]
async fn retries_until_child_succeeds_with_zero_delay_schedule() {
let sup = Supervisor::detached();
let n = Arc::new(AtomicUsize::new(0));
let n_c = Arc::clone(&n);
let out = run_async(
supervised(
&sup,
SupervisorPolicy::Restart {
schedule: Schedule::spaced(duration_const::ZERO),
},
TestClock::new(Instant::now()),
move || {
let c = Arc::clone(&n_c);
let v = c.fetch_add(1, Ordering::SeqCst);
if v < 2 {
fail::<u8, &'static str, ()>("retry")
} else {
succeed::<u8, &'static str, ()>(42)
}
},
),
(),
)
.await
.expect("supervised");
assert_eq!(out, 42);
assert_eq!(n.load(Ordering::SeqCst), 3);
}
#[tokio::test]
async fn escalates_when_schedule_exhausted_with_recurs_zero() {
let sup = Supervisor::detached();
let out = run_async(
supervised(
&sup,
SupervisorPolicy::Restart {
schedule: Schedule::recurs(0),
},
TestClock::new(Instant::now()),
|| fail::<u8, &'static str, ()>("once"),
),
(),
)
.await;
assert_eq!(out, Err(Cause::Fail("once")));
}
}
mod restart_with_limit {
use super::*;
#[tokio::test]
async fn escalates_after_limit_retries_with_then_aggregated_cause() {
let sup = Supervisor::detached();
let clock = TestClock::new(Instant::now());
let n = Arc::new(AtomicUsize::new(0));
let n_c = Arc::clone(&n);
let out = run_async(
supervised(
&sup,
SupervisorPolicy::RestartWithLimit {
limit: 1,
schedule: Schedule::spaced(duration_const::ZERO),
},
clock,
move || {
let c = Arc::clone(&n_c);
let _ = c.fetch_add(1, Ordering::SeqCst);
fail::<u8, &'static str, ()>("e")
},
),
(),
)
.await;
let err = out.expect_err("expected escalation");
match err {
Cause::Then(a, b) => {
assert!(matches!(*a, Cause::Fail("e")));
assert!(matches!(*b, Cause::Fail("e")));
}
other => panic!("unexpected cause: {other:?}"),
}
assert_eq!(n.load(Ordering::SeqCst), 2);
}
#[tokio::test]
async fn tight_restart_loop_stays_bounded_under_high_limit_with_test_clock() {
let sup = Supervisor::detached();
let clock = TestClock::new(Instant::now());
let n = Arc::new(AtomicUsize::new(0));
let n_c = Arc::clone(&n);
let out = run_async(
supervised(
&sup,
SupervisorPolicy::RestartWithLimit {
limit: 50,
schedule: Schedule::spaced(duration_const::ZERO),
},
clock,
move || {
n_c.fetch_add(1, Ordering::SeqCst);
fail::<(), &'static str, ()>("x")
},
),
(),
)
.await;
assert!(out.is_err());
assert_eq!(n.load(Ordering::SeqCst), 51);
}
#[tokio::test]
async fn escalates_when_schedule_exhausted_before_limit_with_recurs_zero() {
let sup = Supervisor::detached();
let out = run_async(
supervised(
&sup,
SupervisorPolicy::RestartWithLimit {
limit: 5,
schedule: Schedule::recurs(0),
},
TestClock::new(Instant::now()),
|| fail::<u8, &'static str, ()>("sched-done"),
),
(),
)
.await;
assert_eq!(out, Err(Cause::Fail("sched-done")));
}
}
mod ignore {
use super::*;
#[tokio::test]
async fn returns_recover_value_when_child_fails() {
let sup = Supervisor::detached();
let out = run_async(
supervised(
&sup,
SupervisorPolicy::Ignore { recover: 99_u8 },
TestClock::new(Instant::now()),
|| fail::<u8, &'static str, ()>("ignored"),
),
(),
)
.await
.expect("ignore");
assert_eq!(out, 99);
}
}
mod escalate {
use super::*;
#[tokio::test]
async fn behaves_like_terminate_for_single_child_failure() {
let sup = Supervisor::detached();
let out = run_async(
supervised(
&sup,
SupervisorPolicy::Escalate,
TestClock::new(Instant::now()),
|| fail::<u8, &'static str, ()>("up"),
),
(),
)
.await;
assert_eq!(out, Err(Cause::Fail("up")));
}
}
}
mod supervisor_from_parts {
use super::*;
#[tokio::test]
async fn from_parts_creates_supervisor_with_given_scope_and_token() {
let scope = Scope::make();
let token = CancellationToken::new();
let sup = Supervisor::from_parts(scope, token.clone());
assert!(!sup.token().is_cancelled());
sup.scope().close();
token.cancel();
assert!(sup.token().is_cancelled());
}
}
mod supervisor_detached_finalizer {
use super::*;
#[test]
fn closing_detached_scope_cancels_token_via_finalizer() {
let sup = Supervisor::detached();
assert!(!sup.token().is_cancelled());
sup.scope().close();
assert!(sup.token().is_cancelled());
}
}
mod supervisor_spawn {
use super::*;
use crate::runtime::ThreadSleepRuntime;
#[test]
fn spawn_runs_supervised_body_on_runtime_worker() {
let rt = ThreadSleepRuntime::default();
let sup = Supervisor::detached();
let h = sup.spawn(
&rt,
SupervisorPolicy::Terminate,
TestClock::new(Instant::now()),
(),
|| succeed::<u8, (), ()>(3),
);
let out = pollster::block_on(h.join());
assert_eq!(out, Ok(3));
}
}
}