use std::collections::VecDeque;
use std::ops::ControlFlow;
use std::sync::Arc;
use tokio::runtime::Handle;
use tokio::sync::mpsc;
use tokio::task::{AbortHandle, JoinHandle};
use crate::actor::panic::{catch_callback, payload_into_string};
use crate::actor::supervision::{
evaluate_strategy, GroupPhase, GroupRestart, ManualStop, StrategyOutcome, SupervisionConfig,
SupervisionState, KILL_GRACE,
};
use crate::actor::{context::ActorContext, handle::ActorHandle, Actor, ActorEnvelope};
use crate::error::{ActorError, SpawnError, SupervisionError};
use crate::system::{ActorSystem, RegistryGuard};
use crate::types::{
ActorId, ActorStatus, ActorStatusInfo, ChildEvent, ChildStoppedInternal, Envelope,
RestartStrategy, RestartType, Shutdown, StopReason, SupervisionAction, SystemMessage,
};
#[derive(Debug, Clone)]
pub struct MailboxConfig {
pub capacity: usize,
}
impl Default for MailboxConfig {
fn default() -> Self {
Self { capacity: 64 }
}
}
impl MailboxConfig {
pub fn with_capacity(mut self, capacity: usize) -> Self {
self.capacity = capacity;
self
}
}
#[derive(Debug, Clone, Default)]
pub struct ActorConfig {
pub mailbox: MailboxConfig,
pub supervision: Option<SupervisionConfig>,
}
impl<'a> From<&'a ActorConfig> for ActorConfig {
fn from(value: &'a ActorConfig) -> Self {
value.clone()
}
}
impl ActorConfig {
pub fn with_mailbox_capacity(mut self, capacity: usize) -> Self {
self.mailbox.capacity = capacity;
self
}
pub fn with_mailbox(mut self, mailbox: MailboxConfig) -> Self {
self.mailbox = mailbox;
self
}
pub fn supervisor(mut self) -> Self {
self.supervision = Some(SupervisionConfig::default());
self
}
pub fn with_supervision(mut self, config: SupervisionConfig) -> Self {
self.supervision = Some(config);
self
}
}
pub(crate) fn into_actor<A: Actor>(
id: impl Into<ActorId>,
actor: A,
config: impl Into<ActorConfig>,
name: Option<String>,
system: Option<Arc<ActorSystem>>,
) -> Result<ActorHandle<A>, SpawnError> {
let (handle, _join) = spawn_actor(id.into(), actor, config.into(), name, system)?;
Ok(handle)
}
pub(crate) fn spawn_actor<A: Actor>(
id: ActorId,
actor: A,
config: ActorConfig,
name: Option<String>,
system: Option<Arc<ActorSystem>>,
) -> Result<(ActorHandle<A>, JoinHandle<StopReason>), SpawnError> {
let handle = Handle::try_current().map_err(|_| SpawnError::MissingRuntime)?;
let mailbox_capacity = config.mailbox.capacity;
let (tx, rx) = mpsc::channel(mailbox_capacity);
let (system_tx, system_rx) = mpsc::channel::<SystemMessage>(64);
let actor_handle = ActorHandle::new(id.clone(), tx, system_tx.clone(), mailbox_capacity);
let supervision = config.supervision.map(SupervisionState::new);
let context = ActorContext::new(
id.clone(),
actor_handle.clone(),
handle.clone(),
system_tx.clone(),
system.clone(),
name.clone(),
supervision,
);
let guard = if name.is_some() || system.is_some() {
let target = system.unwrap_or_else(ActorSystem::default);
target.register_actor::<A>(&id, name.as_deref(), &actor_handle)?;
Some(RegistryGuard::new(target, id, name))
} else {
None
};
let join = handle.spawn(run_actor(actor, context, rx, system_rx, system_tx, guard));
Ok((actor_handle, join))
}
pub(crate) fn spawn_watcher(
child_id: ActorId,
incarnation: u64,
join: JoinHandle<StopReason>,
parent_tx: mpsc::Sender<SystemMessage>,
) -> JoinHandle<()> {
tokio::spawn(async move {
let reason = match join.await {
Ok(reason) => reason,
Err(err) if err.is_panic() => match err.try_into_panic() {
Ok(payload) => StopReason::Failure(ActorError::Panic(payload_into_string(payload))),
Err(_) => {
StopReason::Failure(ActorError::Panic("panic payload unavailable".to_string()))
}
},
Err(_) => StopReason::Cancelled,
};
let _ = parent_tx
.send(SystemMessage::ChildStopped(ChildStoppedInternal {
child_id,
reason,
incarnation,
}))
.await;
})
}
async fn run_actor<A: Actor>(
mut actor: A,
mut ctx: ActorContext<A>,
mut mailbox: mpsc::Receiver<ActorEnvelope<A>>,
mut system_rx: mpsc::Receiver<SystemMessage>,
system_tx: mpsc::Sender<SystemMessage>,
registry_guard: Option<RegistryGuard>,
) -> StopReason {
let phase1 = match catch_callback(actor.pre_start(&mut ctx)).await {
Ok(result) => result,
Err(payload) => Err(ActorError::Panic(payload_into_string(payload))),
};
if let Err(err) = phase1 {
ctx.record_failure(err.clone());
ctx.set_status(ActorStatus::Stopped);
return StopReason::Failure(err);
}
let phase2 = match catch_callback(actor.on_started(&mut ctx)).await {
Ok(result) => result,
Err(payload) => Err(ActorError::Panic(payload_into_string(payload))),
};
if let Err(err) = phase2 {
ctx.record_failure(err.clone());
ctx.set_status(ActorStatus::Stopped);
return StopReason::Failure(err);
}
ctx.set_status(ActorStatus::Running);
let mut stop_reason = StopReason::Graceful;
loop {
tokio::select! {
biased;
sys_msg = system_rx.recv() => {
match sys_msg {
Some(SystemMessage::Stop(reason)) => {
if matches!(reason, StopReason::Kill) {
stop_reason = StopReason::Kill;
break;
}
if matches!(reason, StopReason::Graceful | StopReason::ParentRequest) {
match catch_callback(actor.pre_stop(&reason, &mut ctx)).await {
Ok(true) => {
stop_reason = reason;
break;
}
Ok(false) => continue, Err(payload) => {
stop_reason = StopReason::Failure(ActorError::Panic(
payload_into_string(payload),
));
break;
}
}
}
stop_reason = reason;
break;
}
Some(SystemMessage::GetStatus(reply_tx)) => {
let info = build_status_info(&ctx);
let _ = reply_tx.send(info);
}
Some(SystemMessage::ChildStopped(event)) => {
match handle_child_stopped(&mut actor, &mut ctx, event).await {
ControlFlow::Continue(()) => {}
ControlFlow::Break(reason) => {
stop_reason = reason;
break;
}
}
}
Some(SystemMessage::RestartComplete { seq, child_id, new_system_tx, new_join }) => {
match on_restart_complete(&mut actor, &mut ctx, seq, child_id, new_system_tx, new_join, &system_tx).await {
ControlFlow::Continue(()) => {}
ControlFlow::Break(reason) => {
stop_reason = reason;
break;
}
}
}
None => break, }
}
envelope = mailbox.recv() => {
match envelope {
Some(env) => {
match dispatch(&mut actor, &mut ctx, env).await {
ControlFlow::Continue(()) => {}
ControlFlow::Break(reason) => {
stop_reason = reason;
break;
}
}
}
None => break, }
}
}
}
if !matches!(stop_reason, StopReason::Kill) {
ctx.set_status(ActorStatus::Stopping);
match catch_callback(actor.on_stopped(&stop_reason, &mut ctx)).await {
Ok(Ok(())) => {}
Ok(Err(err)) => {
stop_reason = StopReason::Failure(err);
}
Err(payload) => {
let panic_err = ActorError::Panic(payload_into_string(payload));
ctx.record_failure(panic_err.clone());
if !matches!(stop_reason, StopReason::Failure(_)) {
stop_reason = StopReason::Failure(panic_err);
}
}
}
}
drop(system_rx);
stop_all_children(&mut ctx).await;
ctx.set_status(ActorStatus::Stopped);
#[cfg(feature = "tracing")]
tracing::info!(
actor_id = %ctx.actor_id(),
reason = %stop_reason,
"Actor stopped"
);
drop(registry_guard);
stop_reason
}
async fn dispatch<A: Actor>(
actor: &mut A,
ctx: &mut ActorContext<A>,
envelope: ActorEnvelope<A>,
) -> ControlFlow<StopReason> {
match envelope {
Envelope::Message { payload, responder } => {
match catch_callback(actor.handle(payload, ctx)).await {
Ok(Ok(response)) => {
if let Some(tx) = responder {
let _ = tx.send(Ok(response));
}
ControlFlow::Continue(())
}
Ok(Err(err)) => {
if let Some(tx) = responder {
let _ = tx.send(Err(err.clone()));
ControlFlow::Break(StopReason::Failure(err))
} else {
actor.handle_failure(err);
ControlFlow::Continue(())
}
}
Err(payload) => {
let err = ActorError::Panic(payload_into_string(payload));
if let Some(tx) = responder {
let _ = tx.send(Err(err.clone()));
}
ControlFlow::Break(StopReason::Failure(err))
}
}
}
}
}
async fn handle_child_stopped<A: Actor>(
actor: &mut A,
ctx: &mut ActorContext<A>,
event: ChildStoppedInternal,
) -> ControlFlow<StopReason> {
let mut work: VecDeque<ChildStoppedInternal> = VecDeque::new();
work.push_back(event);
while let Some(ev) = work.pop_front() {
let stale = ctx
.supervision_ref()
.and_then(|sup| sup.registry.get(&ev.child_id))
.map(|child| !child.accepts_incarnation(ev.incarnation))
.unwrap_or(false);
if stale {
continue;
}
let ev = {
let known = ctx
.supervision_ref()
.map(|sup| sup.registry.get(&ev.child_id).is_some())
.unwrap_or(false);
if known && matches!(ev.reason, StopReason::Cancelled) {
ChildStoppedInternal {
reason: StopReason::Kill,
..ev
}
} else {
ev
}
};
let manual = ctx
.supervision_mut()
.and_then(|sup| sup.registry.get_mut(&ev.child_id))
.and_then(|child| {
let taken = child.manual_stop.take();
if taken.is_some() {
child.is_alive = false;
}
taken
});
if let Some(kind) = manual {
let simple = ctx
.supervision_ref()
.map(|sup| matches!(sup.config.strategy, RestartStrategy::SimpleOneForOne))
.unwrap_or(false);
let restart_type = ctx
.supervision_ref()
.and_then(|sup| sup.registry.get(&ev.child_id))
.map(|child| child.spec.restart_type);
let action = match kind {
ManualStop::Terminate => {
prune_spec_if_needed(ctx, &ev.child_id);
SupervisionAction::Removed
}
ManualStop::Bounce if simple => {
if let Some(sup) = ctx.supervision_mut() {
sup.registry.remove(&ev.child_id);
sup.restart_fns.remove(&ev.child_id);
}
SupervisionAction::Removed
}
ManualStop::Bounce if matches!(restart_type, Some(RestartType::Permanent)) => {
if let Some(sup) = ctx.supervision_mut() {
sup.initiate(&ev.child_id);
}
SupervisionAction::RestartInitiated
}
ManualStop::Bounce => {
prune_spec_if_needed(ctx, &ev.child_id);
SupervisionAction::Removed
}
};
let child_name = ctx
.supervision_ref()
.and_then(|sup| sup.registry.get(&ev.child_id))
.and_then(|child| child.name.clone());
let child_event = ChildEvent {
child_id: ev.child_id.clone(),
child_name,
reason: ev.reason.clone(),
action,
};
if let Err(payload) = catch_callback(actor.on_child_stopped(&child_event, ctx)).await {
return ControlFlow::Break(StopReason::Failure(ActorError::Panic(
payload_into_string(payload),
)));
}
continue;
}
let mut member_of_group = false;
let mut retry_front = false;
let mut begin_chain: Option<Vec<ActorId>> = None;
if let Some(sup) = ctx.supervision_mut() {
match sup.pending_group.as_mut() {
Some(GroupPhase::Stopping(group)) => {
if group.awaiting.remove(&ev.child_id) {
member_of_group = true;
if let Some(child) = sup.registry.get_mut(&ev.child_id) {
child.is_alive = false;
}
if group.awaiting.is_empty() {
begin_chain = match sup.pending_group.take() {
Some(GroupPhase::Stopping(group)) => Some(group.restart_order),
_ => None,
};
}
} else {
sup.queued_triggers.push_back(ev);
continue;
}
}
Some(GroupPhase::Restarting(queue)) => {
if queue.front() == Some(&ev.child_id) {
member_of_group = true;
retry_front = true;
if let Some(child) = sup.registry.get_mut(&ev.child_id) {
child.is_alive = false;
}
} else {
sup.queued_triggers.push_back(ev);
continue;
}
}
None => {}
}
}
if member_of_group {
let mut action = SupervisionAction::RestartInitiated;
if retry_front {
let budget_ok = ctx
.supervision_mut()
.map(|sup| sup.budget.check_and_record())
.unwrap_or(false);
if budget_ok {
initiate_restart(ctx, &ev.child_id);
} else {
action = SupervisionAction::BudgetExhausted;
}
} else if let Some(order) = begin_chain {
start_restart_chain(ctx, order, &mut work);
}
let child_name = ctx
.supervision_ref()
.and_then(|sup| sup.registry.get(&ev.child_id))
.and_then(|child| child.name.clone());
let child_event = ChildEvent {
child_id: ev.child_id.clone(),
child_name,
reason: ev.reason.clone(),
action,
};
if let Err(payload) = catch_callback(actor.on_child_stopped(&child_event, ctx)).await {
return ControlFlow::Break(StopReason::Failure(ActorError::Panic(
payload_into_string(payload),
)));
}
if matches!(action, SupervisionAction::BudgetExhausted) {
ctx.record_failure(SupervisionError::BudgetExhausted.into());
return ControlFlow::Break(StopReason::ParentRequest);
}
continue;
}
let child_name = if let Some(sup) = ctx.supervision_mut() {
if let Some(child) = sup.registry.get_mut(&ev.child_id) {
child.is_alive = false;
child.name.clone()
} else {
None
}
} else {
None
};
let outcome = ctx
.supervision_mut()
.map(|sup| evaluate_strategy(sup, &ev.child_id, &ev.reason));
let action = match outcome {
None => SupervisionAction::NotSupervised,
Some(StrategyOutcome::RestartOne(id)) => {
initiate_restart(ctx, &id);
SupervisionAction::RestartInitiated
}
Some(StrategyOutcome::RestartGroup {
stop_reverse,
restart_order,
}) => {
if stop_reverse.is_empty() {
start_restart_chain(ctx, restart_order, &mut work);
} else {
begin_group_stop(ctx, &stop_reverse).await;
if let Some(sup) = ctx.supervision_mut() {
sup.pending_group = Some(GroupPhase::Stopping(GroupRestart {
awaiting: stop_reverse.iter().cloned().collect(),
restart_order,
}));
}
}
SupervisionAction::RestartInitiated
}
Some(StrategyOutcome::Remove) => {
prune_spec_if_needed(ctx, &ev.child_id);
SupervisionAction::Removed
}
Some(StrategyOutcome::BudgetExhausted) => SupervisionAction::BudgetExhausted,
};
let child_event = ChildEvent {
child_id: ev.child_id.clone(),
child_name,
reason: ev.reason.clone(),
action,
};
if let Err(payload) = catch_callback(actor.on_child_stopped(&child_event, ctx)).await {
return ControlFlow::Break(StopReason::Failure(ActorError::Panic(
payload_into_string(payload),
)));
}
if matches!(action, SupervisionAction::BudgetExhausted) {
ctx.record_failure(SupervisionError::BudgetExhausted.into());
return ControlFlow::Break(StopReason::ParentRequest);
}
}
ControlFlow::Continue(())
}
fn start_restart_chain<A: Actor>(
ctx: &mut ActorContext<A>,
order: Vec<ActorId>,
work: &mut VecDeque<ChildStoppedInternal>,
) {
let queue: VecDeque<ActorId> = order.into();
match queue.front().cloned() {
Some(front) => {
if let Some(sup) = ctx.supervision_mut() {
sup.pending_group = Some(GroupPhase::Restarting(queue));
}
initiate_restart(ctx, &front);
}
None => {
if let Some(sup) = ctx.supervision_mut() {
sup.pending_group = None;
while let Some(queued) = sup.queued_triggers.pop_front() {
work.push_back(queued);
}
}
}
}
}
async fn on_restart_complete<A: Actor>(
actor: &mut A,
ctx: &mut ActorContext<A>,
seq: u64,
child_id: ActorId,
new_system_tx: mpsc::Sender<SystemMessage>,
new_join: JoinHandle<StopReason>,
system_tx: &mpsc::Sender<SystemMessage>,
) -> ControlFlow<StopReason> {
let accepted = ctx
.supervision_ref()
.and_then(|sup| sup.registry.get(&child_id))
.map(|child| child.pending_restart_seq == Some(seq))
.unwrap_or(false);
if !accepted {
let _ = new_system_tx
.send(SystemMessage::Stop(StopReason::Kill))
.await;
drop(new_join);
return ControlFlow::Continue(());
}
let abort = new_join.abort_handle();
let watcher = spawn_watcher(child_id.clone(), seq, new_join, system_tx.clone());
let mut next_in_chain: Option<ActorId> = None;
let mut drained: Vec<ChildStoppedInternal> = Vec::new();
if let Some(sup) = ctx.supervision_mut() {
sup.registry
.update_restarted(&child_id, seq, new_system_tx, watcher, abort);
if let Some(GroupPhase::Restarting(queue)) = sup.pending_group.as_mut() {
if queue.front() == Some(&child_id) {
queue.pop_front();
match queue.front().cloned() {
Some(next) => next_in_chain = Some(next),
None => {
sup.pending_group = None;
drained.extend(sup.queued_triggers.drain(..));
}
}
}
}
}
if let Some(next) = next_in_chain {
initiate_restart(ctx, &next);
}
for ev in drained {
if let ControlFlow::Break(reason) = handle_child_stopped(actor, ctx, ev).await {
return ControlFlow::Break(reason);
}
}
ControlFlow::Continue(())
}
async fn begin_group_stop<A: Actor>(ctx: &mut ActorContext<A>, stop_reverse: &[ActorId]) {
let mut targets: Vec<(mpsc::Sender<SystemMessage>, Shutdown, AbortHandle)> = Vec::new();
if let Some(sup) = ctx.supervision_ref() {
for id in stop_reverse {
if let Some(child) = sup.registry.get(id) {
targets.push((
child.system_tx.clone(),
child.spec.shutdown,
child.abort.clone(),
));
}
}
}
for (tx, shutdown, abort) in targets {
match shutdown {
Shutdown::Kill => {
let _ = tx.send(SystemMessage::Stop(StopReason::Kill)).await;
spawn_grace_abort(abort);
}
Shutdown::Timeout(after) => {
let _ = tx
.send(SystemMessage::Stop(StopReason::ParentRequest))
.await;
let escalate = tx.clone();
tokio::spawn(async move {
tokio::time::sleep(after).await;
let _ = escalate.send(SystemMessage::Stop(StopReason::Kill)).await;
tokio::time::sleep(KILL_GRACE).await;
abort.abort();
});
}
Shutdown::Infinity => {
let _ = tx
.send(SystemMessage::Stop(StopReason::ParentRequest))
.await;
}
}
}
}
fn spawn_grace_abort(abort: AbortHandle) {
tokio::spawn(async move {
tokio::time::sleep(KILL_GRACE).await;
abort.abort();
});
}
fn prune_spec_if_needed<A: Actor>(ctx: &mut ActorContext<A>, child_id: &ActorId) {
if let Some(sup) = ctx.supervision_mut() {
let prune = matches!(sup.config.strategy, RestartStrategy::SimpleOneForOne)
|| sup
.registry
.get(child_id)
.map(|c| matches!(c.spec.restart_type, RestartType::Temporary))
.unwrap_or(false);
if prune {
sup.registry.remove(child_id);
sup.restart_fns.remove(child_id);
}
}
}
fn initiate_restart<A: Actor>(ctx: &mut ActorContext<A>, child_id: &ActorId) {
if let Some(sup) = ctx.supervision_mut() {
sup.initiate(child_id);
}
}
async fn stop_all_children<A: Actor>(ctx: &mut ActorContext<A>) {
let children = match ctx.supervision_mut() {
Some(sup) => {
sup.restart_fns.clear();
sup.pending_group = None;
sup.queued_triggers.clear();
sup.registry.drain_all()
}
None => return,
};
for child in children.into_iter().rev() {
if !child.is_alive {
continue;
}
let mut watcher = child.watcher_handle;
let abort = child.abort;
match child.spec.shutdown {
Shutdown::Kill => {
let _ = child
.system_tx
.send(SystemMessage::Stop(StopReason::Kill))
.await;
if tokio::time::timeout(KILL_GRACE, &mut watcher)
.await
.is_err()
{
abort.abort();
let _ = tokio::time::timeout(KILL_GRACE, &mut watcher).await;
}
}
Shutdown::Timeout(after) => {
let _ = child
.system_tx
.send(SystemMessage::Stop(StopReason::ParentRequest))
.await;
if tokio::time::timeout(after, &mut watcher).await.is_err() {
let _ = child
.system_tx
.send(SystemMessage::Stop(StopReason::Kill))
.await;
if tokio::time::timeout(KILL_GRACE, &mut watcher)
.await
.is_err()
{
abort.abort();
let _ = tokio::time::timeout(KILL_GRACE, &mut watcher).await;
}
}
}
Shutdown::Infinity => {
let _ = child
.system_tx
.send(SystemMessage::Stop(StopReason::ParentRequest))
.await;
let _ = watcher.await;
}
}
}
}
fn build_status_info<A: Actor>(ctx: &ActorContext<A>) -> ActorStatusInfo {
let (child_count, name) = ctx
.supervision_ref()
.map(|sup| (sup.registry.len(), ctx.actor_name().cloned()))
.unwrap_or((0, ctx.actor_name().cloned()));
ActorStatusInfo {
id: ctx.actor_id().clone(),
name,
status: ctx.status(),
mailbox_len: ctx.self_handle().mailbox_len(),
mailbox_capacity: ctx.self_handle().mailbox_capacity(),
child_count,
timer_count: ctx.active_timer_count(),
stream_count: ctx.active_stream_count(),
}
}