use std::{collections::BTreeMap, error::Error as StdError, fmt, sync::Arc, time::Duration};
use async_trait::async_trait;
use thiserror::Error;
use tokio::time::{self, Instant};
use crate::result::{BoxError, ClassifyError, ErrorKind};
const MAX_LIFECYCLE_HOOK_NAME_LENGTH: usize = 96;
pub const DEFAULT_LIFECYCLE_HOOK_TIMEOUT: Duration = Duration::from_secs(30);
pub const DEFAULT_LIFECYCLE_STAGE_TIMEOUT: Duration = Duration::from_secs(120);
pub const DEFAULT_LIFECYCLE_CLEANUP_TIMEOUT: Duration = Duration::from_secs(120);
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct CompletedStartupAction {
hook_index: usize,
stage: LifecycleStage,
}
#[derive(Clone, Default)]
pub struct Lifecycle {
hooks: Arc<[Arc<dyn LifecycleHook>]>,
timeouts: LifecycleTimeouts,
}
impl Lifecycle {
#[must_use]
pub fn builder() -> LifecycleBuilder {
LifecycleBuilder::new()
}
#[must_use]
pub fn hooks(&self) -> &[Arc<dyn LifecycleHook>] {
&self.hooks
}
#[must_use]
pub const fn timeouts(&self) -> LifecycleTimeouts {
self.timeouts
}
pub async fn run(&self, stage: LifecycleStage) -> Result<LifecycleRunSummary, LifecycleError> {
self.run_with_timeouts(stage, self.timeouts).await
}
pub async fn run_with_timeouts(
&self,
stage: LifecycleStage,
timeouts: LifecycleTimeouts,
) -> Result<LifecycleRunSummary, LifecycleError> {
self.run_inner(stage, Some(timeouts), timeouts).await
}
pub async fn run_unbounded(
&self,
stage: LifecycleStage,
) -> Result<LifecycleRunSummary, LifecycleError> {
self.run_inner(stage, None, self.timeouts).await
}
#[must_use = "a startup sequence must be advanced or explicitly aborted"]
pub fn startup(&self) -> LifecycleStartup {
self.startup_with_timeouts(self.timeouts)
}
#[must_use = "a startup sequence must be advanced or explicitly aborted"]
pub fn startup_with_timeouts(&self, timeouts: LifecycleTimeouts) -> LifecycleStartup {
LifecycleStartup {
lifecycle: self.clone(),
timeouts,
next_stage: Some(LifecycleStage::BeforeStart),
completed: Vec::with_capacity(self.hooks.len().saturating_mul(2)),
}
}
async fn run_inner(
&self,
stage: LifecycleStage,
timeouts: Option<LifecycleTimeouts>,
cleanup_timeouts: LifecycleTimeouts,
) -> Result<LifecycleRunSummary, LifecycleError> {
if stage.is_shutdown() {
let stage_deadline = match stage_deadline(timeouts) {
Ok(deadline) => deadline,
Err(failure) => {
return Err(LifecycleError::new(stage, vec![failure], Vec::new()));
}
};
let mut failures = Vec::new();
for hook in self.hooks.iter().rev() {
if let Err(failure) = run_lifecycle_hook(
hook,
stage,
LifecycleOperation::Run,
timeouts,
stage_deadline.map(OverallDeadline::Stage),
|| {},
)
.await
{
failures.push(failure);
}
}
return if failures.is_empty() {
Ok(LifecycleRunSummary::new(stage, self.hooks.len()))
} else {
Err(LifecycleError::new(stage, failures, Vec::new()))
};
}
let mut completed = Vec::with_capacity(self.hooks.len());
self.run_startup_stage(stage, timeouts, cleanup_timeouts, &mut completed)
.await
}
async fn run_startup_stage(
&self,
stage: LifecycleStage,
timeouts: Option<LifecycleTimeouts>,
cleanup_timeouts: LifecycleTimeouts,
completed: &mut Vec<CompletedStartupAction>,
) -> Result<LifecycleRunSummary, LifecycleError> {
debug_assert!(!stage.is_shutdown());
let stage_deadline = match stage_deadline(timeouts) {
Ok(deadline) => deadline,
Err(failure) => {
let cleanup_failures = self
.rollback_startup_actions(completed, cleanup_timeouts)
.await;
return Err(LifecycleError::new(stage, vec![failure], cleanup_failures));
}
};
for (hook_index, hook) in self.hooks.iter().enumerate() {
if let Err(failure) = run_lifecycle_hook(
hook,
stage,
LifecycleOperation::Run,
timeouts,
stage_deadline.map(OverallDeadline::Stage),
|| {
completed.push(CompletedStartupAction { hook_index, stage });
},
)
.await
{
let cleanup_failures = self
.rollback_startup_actions(completed, cleanup_timeouts)
.await;
return Err(LifecycleError::new(stage, vec![failure], cleanup_failures));
}
}
Ok(LifecycleRunSummary::new(stage, self.hooks.len()))
}
async fn rollback_startup_actions(
&self,
actions: &mut Vec<CompletedStartupAction>,
timeouts: LifecycleTimeouts,
) -> Vec<LifecycleFailure> {
let cleanup_deadline = match cleanup_deadline(timeouts) {
Ok(deadline) => deadline,
Err(()) => {
return actions
.iter()
.rev()
.map(|action| {
LifecycleFailure::deadline_unavailable(
LifecycleOperation::Rollback,
Some(self.hooks[action.hook_index].name()),
)
})
.collect();
}
};
let mut failures = Vec::new();
for index in (0..actions.len()).rev() {
let action = actions[index];
let hook = &self.hooks[action.hook_index];
match run_lifecycle_hook(
hook,
action.stage,
LifecycleOperation::Rollback,
Some(timeouts),
Some(OverallDeadline::Cleanup(cleanup_deadline)),
|| {},
)
.await
{
Ok(()) => {
actions.remove(index);
}
Err(failure) => failures.push(failure),
}
}
failures
}
}
#[must_use = "a startup sequence must be advanced or explicitly aborted"]
pub struct LifecycleStartup {
lifecycle: Lifecycle,
timeouts: LifecycleTimeouts,
next_stage: Option<LifecycleStage>,
completed: Vec<CompletedStartupAction>,
}
impl LifecycleStartup {
#[must_use]
pub const fn next_stage(&self) -> Option<LifecycleStage> {
self.next_stage
}
pub async fn run_next(&mut self) -> Result<Option<LifecycleRunSummary>, LifecycleError> {
let Some(stage) = self.next_stage.take() else {
return Ok(None);
};
let summary = self
.lifecycle
.run_startup_stage(
stage,
Some(self.timeouts),
self.timeouts,
&mut self.completed,
)
.await?;
if stage == LifecycleStage::BeforeStart {
self.next_stage = Some(LifecycleStage::AfterStart);
} else {
self.completed.clear();
}
Ok(Some(summary))
}
pub async fn abort(&mut self) -> Result<(), LifecycleError> {
self.next_stage = None;
let Some(last) = self.completed.last().copied() else {
return Ok(());
};
let cleanup_failures = self
.lifecycle
.rollback_startup_actions(&mut self.completed, self.timeouts)
.await;
if cleanup_failures.is_empty() {
Ok(())
} else {
Err(LifecycleError::new(
last.stage,
Vec::new(),
cleanup_failures,
))
}
}
}
impl fmt::Debug for LifecycleStartup {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("LifecycleStartup")
.field("next_stage", &self.next_stage)
.field("retained_actions", &self.completed.len())
.field("timeouts", &self.timeouts)
.finish()
}
}
impl fmt::Debug for Lifecycle {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("Lifecycle")
.field("hooks", &self.hooks.len())
.field("timeouts", &self.timeouts)
.finish()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub struct LifecycleTimeouts {
hook_timeout: Duration,
stage_timeout: Duration,
cleanup_timeout: Duration,
}
impl LifecycleTimeouts {
pub fn new(
hook_timeout: Duration,
stage_timeout: Duration,
cleanup_timeout: Duration,
) -> Result<Self, LifecycleTimeoutError> {
if hook_timeout.is_zero() || Instant::now().checked_add(hook_timeout).is_none() {
return Err(LifecycleTimeoutError::InvalidHookTimeout);
}
if stage_timeout.is_zero() || Instant::now().checked_add(stage_timeout).is_none() {
return Err(LifecycleTimeoutError::InvalidStageTimeout);
}
if cleanup_timeout.is_zero() || Instant::now().checked_add(cleanup_timeout).is_none() {
return Err(LifecycleTimeoutError::InvalidCleanupTimeout);
}
if hook_timeout > stage_timeout {
return Err(LifecycleTimeoutError::HookExceedsStage);
}
if hook_timeout > cleanup_timeout {
return Err(LifecycleTimeoutError::HookExceedsCleanup);
}
Ok(Self {
hook_timeout,
stage_timeout,
cleanup_timeout,
})
}
#[must_use]
pub const fn hook_timeout(self) -> Duration {
self.hook_timeout
}
#[must_use]
pub const fn stage_timeout(self) -> Duration {
self.stage_timeout
}
#[must_use]
pub const fn cleanup_timeout(self) -> Duration {
self.cleanup_timeout
}
}
impl Default for LifecycleTimeouts {
fn default() -> Self {
Self {
hook_timeout: DEFAULT_LIFECYCLE_HOOK_TIMEOUT,
stage_timeout: DEFAULT_LIFECYCLE_STAGE_TIMEOUT,
cleanup_timeout: DEFAULT_LIFECYCLE_CLEANUP_TIMEOUT,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]
#[non_exhaustive]
pub enum LifecycleTimeoutError {
#[error("lifecycle hook timeout must be non-zero and fit the runtime timer range")]
InvalidHookTimeout,
#[error("lifecycle stage timeout must be non-zero and fit the runtime timer range")]
InvalidStageTimeout,
#[error("lifecycle cleanup timeout must be non-zero and fit the runtime timer range")]
InvalidCleanupTimeout,
#[error("lifecycle hook timeout must not exceed the stage timeout")]
HookExceedsStage,
#[error("lifecycle hook timeout must not exceed the cleanup timeout")]
HookExceedsCleanup,
}
impl ClassifyError for LifecycleTimeoutError {
fn error_kind(&self) -> ErrorKind {
ErrorKind::Validation
}
fn error_code(&self) -> crate::result::ErrorCode {
crate::result::ErrorCode::internal(match self {
Self::InvalidHookTimeout => "lifecycle_invalid_hook_timeout",
Self::InvalidStageTimeout => "lifecycle_invalid_stage_timeout",
Self::InvalidCleanupTimeout => "lifecycle_invalid_cleanup_timeout",
Self::HookExceedsStage => "lifecycle_hook_timeout_exceeds_stage",
Self::HookExceedsCleanup => "lifecycle_hook_timeout_exceeds_cleanup",
})
}
}
#[derive(Clone, Default)]
pub struct LifecycleBuilder {
hooks: Vec<Arc<dyn LifecycleHook>>,
timeouts: LifecycleTimeouts,
}
impl LifecycleBuilder {
#[must_use]
pub fn new() -> Self {
Self::default()
}
#[must_use]
pub fn hook(mut self, hook: Arc<dyn LifecycleHook>) -> Self {
self.hooks.push(hook);
self
}
#[must_use]
pub const fn timeouts(mut self, timeouts: LifecycleTimeouts) -> Self {
self.timeouts = timeouts;
self
}
pub fn build(self) -> Result<Lifecycle, LifecycleBuildError> {
validate_hooks(&self.hooks)?;
Ok(Lifecycle {
hooks: Arc::from(self.hooks),
timeouts: self.timeouts,
})
}
}
impl fmt::Debug for LifecycleBuilder {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("LifecycleBuilder")
.field("hooks", &self.hooks.len())
.field("timeouts", &self.timeouts)
.finish()
}
}
#[async_trait]
pub trait LifecycleHook: Send + Sync + 'static {
fn name(&self) -> &'static str;
async fn run(&self, stage: LifecycleStage) -> Result<(), BoxError>;
async fn rollback(&self, _startup_stage: LifecycleStage) -> Result<(), BoxError> {
Ok(())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub enum LifecycleStage {
BeforeStart,
AfterStart,
BeforeShutdown,
AfterShutdown,
}
impl LifecycleStage {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::BeforeStart => "before_start",
Self::AfterStart => "after_start",
Self::BeforeShutdown => "before_shutdown",
Self::AfterShutdown => "after_shutdown",
}
}
const fn is_shutdown(self) -> bool {
matches!(self, Self::BeforeShutdown | Self::AfterShutdown)
}
}
impl fmt::Display for LifecycleStage {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(self.as_str())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub struct LifecycleRunSummary {
stage: LifecycleStage,
hooks: usize,
}
impl LifecycleRunSummary {
const fn new(stage: LifecycleStage, hooks: usize) -> Self {
Self { stage, hooks }
}
#[must_use]
pub const fn stage(self) -> LifecycleStage {
self.stage
}
#[must_use]
pub const fn hooks(self) -> usize {
self.hooks
}
}
#[derive(Debug, Clone, PartialEq, Eq, Error)]
#[non_exhaustive]
pub enum LifecycleBuildError {
#[error("lifecycle hook name at index {index} is invalid: {name:?}")]
InvalidHookName {
index: usize,
name: &'static str,
},
#[error(
"lifecycle hook name {name:?} is duplicated at indexes {first_index} and {second_index}"
)]
DuplicateHookName {
first_index: usize,
second_index: usize,
name: &'static str,
},
}
impl ClassifyError for LifecycleBuildError {
fn error_kind(&self) -> ErrorKind {
ErrorKind::Validation
}
fn error_code(&self) -> crate::result::ErrorCode {
crate::result::ErrorCode::internal(match self {
Self::InvalidHookName { .. } => "lifecycle_invalid_hook_name",
Self::DuplicateHookName { .. } => "lifecycle_duplicate_hook_name",
})
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum LifecycleOperation {
Run,
Rollback,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum LifecycleFailureKind {
HookFailed,
HookTimedOut,
StageTimedOut,
CleanupTimedOut,
DeadlineUnavailable,
}
pub struct LifecycleFailure {
hook: Option<&'static str>,
operation: LifecycleOperation,
kind: LifecycleFailureKind,
source: Option<BoxError>,
}
impl LifecycleFailure {
fn hook_failed(hook: &'static str, operation: LifecycleOperation, source: BoxError) -> Self {
Self {
hook: Some(hook),
operation,
kind: LifecycleFailureKind::HookFailed,
source: Some(source),
}
}
const fn timed_out(
hook: &'static str,
operation: LifecycleOperation,
kind: LifecycleFailureKind,
) -> Self {
Self {
hook: Some(hook),
operation,
kind,
source: None,
}
}
const fn deadline_unavailable(
operation: LifecycleOperation,
hook: Option<&'static str>,
) -> Self {
Self {
hook,
operation,
kind: LifecycleFailureKind::DeadlineUnavailable,
source: None,
}
}
#[must_use]
pub const fn hook(&self) -> Option<&'static str> {
self.hook
}
#[must_use]
pub const fn operation(&self) -> LifecycleOperation {
self.operation
}
#[must_use]
pub const fn kind(&self) -> LifecycleFailureKind {
self.kind
}
#[must_use]
pub fn source_error(&self) -> Option<&(dyn StdError + Send + Sync + 'static)> {
self.source.as_deref()
}
}
impl fmt::Debug for LifecycleFailure {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("LifecycleFailure")
.field("hook", &self.hook)
.field("operation", &self.operation)
.field("kind", &self.kind)
.finish_non_exhaustive()
}
}
pub struct LifecycleError {
stage: LifecycleStage,
failures: Vec<LifecycleFailure>,
cleanup_failures: Vec<LifecycleFailure>,
}
impl LifecycleError {
fn new(
stage: LifecycleStage,
failures: Vec<LifecycleFailure>,
cleanup_failures: Vec<LifecycleFailure>,
) -> Self {
Self {
stage,
failures,
cleanup_failures,
}
}
#[must_use]
pub const fn stage(&self) -> LifecycleStage {
self.stage
}
#[must_use]
pub fn failures(&self) -> &[LifecycleFailure] {
&self.failures
}
#[must_use]
pub fn cleanup_failures(&self) -> &[LifecycleFailure] {
&self.cleanup_failures
}
#[must_use]
pub fn timed_out(&self) -> bool {
self.failures
.iter()
.chain(&self.cleanup_failures)
.any(|failure| {
matches!(
failure.kind,
LifecycleFailureKind::HookTimedOut
| LifecycleFailureKind::StageTimedOut
| LifecycleFailureKind::CleanupTimedOut
)
})
}
fn code(&self) -> &'static str {
if !self.cleanup_failures.is_empty() {
return "lifecycle_startup_rollback_failed";
}
if self.stage.is_shutdown() && self.failures.len() > 1 {
return "lifecycle_shutdown_failed";
}
match self.failures.first().map(|failure| failure.kind) {
Some(LifecycleFailureKind::HookFailed) => "lifecycle_hook_failed",
Some(LifecycleFailureKind::HookTimedOut) => "lifecycle_hook_timed_out",
Some(LifecycleFailureKind::StageTimedOut) => "lifecycle_stage_timed_out",
Some(LifecycleFailureKind::CleanupTimedOut) => "lifecycle_cleanup_timed_out",
Some(LifecycleFailureKind::DeadlineUnavailable) | None => {
"lifecycle_deadline_unavailable"
}
}
}
}
impl fmt::Display for LifecycleError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
formatter,
"lifecycle stage {} failed with {} hook failure(s) and {} cleanup failure(s)",
self.stage,
self.failures.len(),
self.cleanup_failures.len()
)
}
}
impl fmt::Debug for LifecycleError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("LifecycleError")
.field("code", &self.code())
.field("stage", &self.stage)
.field("failures", &self.failures)
.field("cleanup_failures", &self.cleanup_failures)
.finish_non_exhaustive()
}
}
impl StdError for LifecycleError {
fn source(&self) -> Option<&(dyn StdError + 'static)> {
self.failures
.iter()
.chain(&self.cleanup_failures)
.find_map(|failure| {
failure
.source_error()
.map(|source| source as &(dyn StdError + 'static))
})
}
}
impl ClassifyError for LifecycleError {
fn error_kind(&self) -> ErrorKind {
if self
.failures
.iter()
.chain(&self.cleanup_failures)
.any(|failure| failure.kind == LifecycleFailureKind::HookFailed)
{
return ErrorKind::Internal;
}
if self
.failures
.iter()
.chain(&self.cleanup_failures)
.any(|failure| failure.kind == LifecycleFailureKind::DeadlineUnavailable)
{
return ErrorKind::Unavailable;
}
ErrorKind::Timeout
}
fn error_code(&self) -> crate::result::ErrorCode {
crate::result::ErrorCode::internal(self.code())
}
}
#[derive(Clone, Copy)]
enum OverallDeadline {
Stage(Instant),
Cleanup(Instant),
}
impl OverallDeadline {
const fn instant(self) -> Instant {
match self {
Self::Stage(deadline) | Self::Cleanup(deadline) => deadline,
}
}
const fn timeout_kind(self) -> LifecycleFailureKind {
match self {
Self::Stage(_) => LifecycleFailureKind::StageTimedOut,
Self::Cleanup(_) => LifecycleFailureKind::CleanupTimedOut,
}
}
}
fn stage_deadline(
timeouts: Option<LifecycleTimeouts>,
) -> Result<Option<Instant>, LifecycleFailure> {
timeouts
.map(|timeouts| {
Instant::now()
.checked_add(timeouts.stage_timeout)
.ok_or_else(|| {
LifecycleFailure::deadline_unavailable(LifecycleOperation::Run, None)
})
})
.transpose()
}
fn cleanup_deadline(timeouts: LifecycleTimeouts) -> Result<Instant, ()> {
Instant::now()
.checked_add(timeouts.cleanup_timeout)
.ok_or(())
}
async fn run_lifecycle_hook(
hook: &Arc<dyn LifecycleHook>,
stage: LifecycleStage,
operation: LifecycleOperation,
timeouts: Option<LifecycleTimeouts>,
overall_deadline: Option<OverallDeadline>,
on_start: impl FnOnce(),
) -> Result<(), LifecycleFailure> {
tracing::debug!(
lifecycle_stage = stage.as_str(),
lifecycle_hook = hook.name(),
lifecycle_operation = ?operation,
"running lifecycle hook"
);
let run = async {
on_start();
match operation {
LifecycleOperation::Run => hook.run(stage).await,
LifecycleOperation::Rollback => hook.rollback(stage).await,
}
};
let Some(timeouts) = timeouts else {
return run
.await
.map_err(|source| LifecycleFailure::hook_failed(hook.name(), operation, source));
};
let Some(overall_deadline) = overall_deadline else {
return Err(LifecycleFailure::deadline_unavailable(
operation,
Some(hook.name()),
));
};
let now = Instant::now();
let overall_instant = overall_deadline.instant();
if now >= overall_instant {
return Err(LifecycleFailure::timed_out(
hook.name(),
operation,
overall_deadline.timeout_kind(),
));
}
let hook_deadline = now
.checked_add(timeouts.hook_timeout)
.ok_or_else(|| LifecycleFailure::deadline_unavailable(operation, Some(hook.name())))?;
let (deadline, timeout_kind) = if hook_deadline <= overall_instant {
(hook_deadline, LifecycleFailureKind::HookTimedOut)
} else {
(overall_instant, overall_deadline.timeout_kind())
};
match time::timeout_at(deadline, run).await {
Ok(Ok(())) => Ok(()),
Ok(Err(source)) => Err(LifecycleFailure::hook_failed(
hook.name(),
operation,
source,
)),
Err(_) => Err(LifecycleFailure::timed_out(
hook.name(),
operation,
timeout_kind,
)),
}
}
fn validate_hooks(hooks: &[Arc<dyn LifecycleHook>]) -> Result<(), LifecycleBuildError> {
let mut names = BTreeMap::<&'static str, usize>::new();
for (index, hook) in hooks.iter().enumerate() {
let name = hook.name();
if !valid_hook_name(name) {
return Err(LifecycleBuildError::InvalidHookName { index, name });
}
if let Some(first_index) = names.insert(name, index) {
return Err(LifecycleBuildError::DuplicateHookName {
first_index,
second_index: index,
name,
});
}
}
Ok(())
}
fn valid_hook_name(value: &str) -> bool {
!value.is_empty()
&& value.len() <= MAX_LIFECYCLE_HOOK_NAME_LENGTH
&& value
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b':'))
}
#[cfg(test)]
mod tests {
use std::sync::Mutex;
use super::*;
#[tokio::test]
async fn lifecycle_runs_startup_hooks_in_registration_order() {
let seen = Arc::new(Mutex::new(Vec::new()));
let lifecycle = Lifecycle::builder()
.hook(Arc::new(RecordingHook::new("first", Arc::clone(&seen))))
.hook(Arc::new(RecordingHook::new("second", Arc::clone(&seen))))
.build()
.expect("lifecycle should build");
let summary = lifecycle
.run(LifecycleStage::BeforeStart)
.await
.expect("lifecycle should run");
assert_eq!(summary.hooks(), 2);
assert_eq!(&*seen.lock().unwrap(), &["first", "second"]);
}
#[tokio::test]
async fn lifecycle_runs_shutdown_hooks_in_reverse_registration_order() {
let seen = Arc::new(Mutex::new(Vec::new()));
let lifecycle = Lifecycle::builder()
.hook(Arc::new(RecordingHook::new(
"dependency",
Arc::clone(&seen),
)))
.hook(Arc::new(RecordingHook::new("dependent", Arc::clone(&seen))))
.build()
.expect("lifecycle should build");
lifecycle
.run(LifecycleStage::BeforeShutdown)
.await
.expect("lifecycle should run");
assert_eq!(&*seen.lock().unwrap(), &["dependent", "dependency"]);
}
#[test]
fn lifecycle_rejects_duplicate_hook_names() {
let seen = Arc::new(Mutex::new(Vec::new()));
let error = Lifecycle::builder()
.hook(Arc::new(RecordingHook::new("same", Arc::clone(&seen))))
.hook(Arc::new(RecordingHook::new("same", Arc::clone(&seen))))
.build()
.expect_err("duplicate hooks must be rejected");
assert!(matches!(
error,
LifecycleBuildError::DuplicateHookName {
first_index: 0,
second_index: 1,
..
}
));
}
#[tokio::test]
async fn lifecycle_reports_hook_failure_without_exposing_source_details() {
let lifecycle = Lifecycle::builder()
.hook(Arc::new(FailingHook))
.build()
.expect("lifecycle should build");
let error = lifecycle
.run(LifecycleStage::BeforeStart)
.await
.expect_err("hook should fail");
assert!(!format!("{error:?}").contains("private failure"));
assert!(!error.to_string().contains("private failure"));
}
#[tokio::test]
async fn startup_failure_rolls_back_failing_then_completed_hooks_in_reverse_order() {
let seen = Arc::new(Mutex::new(Vec::new()));
let lifecycle = Lifecycle::builder()
.hook(Arc::new(ControlledHook::new(
"first",
Arc::clone(&seen),
HookBehavior::Succeed,
HookBehavior::Succeed,
)))
.hook(Arc::new(ControlledHook::new(
"second",
Arc::clone(&seen),
HookBehavior::Succeed,
HookBehavior::Succeed,
)))
.hook(Arc::new(ControlledHook::new(
"third",
Arc::clone(&seen),
HookBehavior::Fail,
HookBehavior::Succeed,
)))
.build()
.expect("lifecycle should build");
let error = lifecycle
.run(LifecycleStage::BeforeStart)
.await
.expect_err("startup should fail");
assert_eq!(error.failures().len(), 1);
assert!(error.cleanup_failures().is_empty());
assert_eq!(error.failures()[0].hook(), Some("third"));
assert_eq!(
&*seen.lock().unwrap(),
&[
"run:first",
"run:second",
"run:third",
"rollback:third",
"rollback:second",
"rollback:first",
]
);
}
#[tokio::test]
async fn startup_preserves_primary_and_cleanup_failures_without_source_details() {
let seen = Arc::new(Mutex::new(Vec::new()));
let lifecycle = Lifecycle::builder()
.hook(Arc::new(ControlledHook::new(
"first",
Arc::clone(&seen),
HookBehavior::Succeed,
HookBehavior::Fail,
)))
.hook(Arc::new(ControlledHook::new(
"second",
Arc::clone(&seen),
HookBehavior::Fail,
HookBehavior::Fail,
)))
.build()
.expect("lifecycle should build");
let error = lifecycle
.run(LifecycleStage::AfterStart)
.await
.expect_err("startup and rollback should fail");
assert_eq!(error.failures().len(), 1);
assert_eq!(error.cleanup_failures().len(), 2);
assert!(
error
.cleanup_failures()
.iter()
.all(|failure| failure.operation() == LifecycleOperation::Rollback)
);
assert_eq!(error.cleanup_failures()[0].hook(), Some("second"));
assert_eq!(error.cleanup_failures()[1].hook(), Some("first"));
let debug = format!("{error:?}");
assert!(!debug.contains("private run failure"));
assert!(!debug.contains("private rollback failure"));
assert_eq!(error.error_code(), "lifecycle_startup_rollback_failed");
assert_eq!(
&*seen.lock().unwrap(),
&[
"run:first",
"run:second",
"rollback:second",
"rollback:first"
]
);
}
#[tokio::test]
async fn shutdown_runs_every_hook_in_reverse_and_aggregates_failures() {
let seen = Arc::new(Mutex::new(Vec::new()));
let lifecycle = Lifecycle::builder()
.hook(Arc::new(ControlledHook::new(
"first",
Arc::clone(&seen),
HookBehavior::Fail,
HookBehavior::Succeed,
)))
.hook(Arc::new(ControlledHook::new(
"second",
Arc::clone(&seen),
HookBehavior::Succeed,
HookBehavior::Succeed,
)))
.hook(Arc::new(ControlledHook::new(
"third",
Arc::clone(&seen),
HookBehavior::Fail,
HookBehavior::Succeed,
)))
.build()
.expect("lifecycle should build");
let error = lifecycle
.run(LifecycleStage::BeforeShutdown)
.await
.expect_err("shutdown failures should be aggregated");
assert_eq!(error.failures().len(), 2);
assert!(error.cleanup_failures().is_empty());
assert_eq!(error.error_code(), "lifecycle_shutdown_failed");
assert_eq!(
&*seen.lock().unwrap(),
&["run:third", "run:second", "run:first"]
);
}
#[tokio::test]
async fn shutdown_continues_after_a_hung_hook_times_out() {
let seen = Arc::new(Mutex::new(Vec::new()));
let lifecycle = Lifecycle::builder()
.hook(Arc::new(ControlledHook::new(
"first",
Arc::clone(&seen),
HookBehavior::Succeed,
HookBehavior::Succeed,
)))
.hook(Arc::new(ControlledHook::new(
"hung",
Arc::clone(&seen),
HookBehavior::Hang,
HookBehavior::Succeed,
)))
.hook(Arc::new(ControlledHook::new(
"third",
Arc::clone(&seen),
HookBehavior::Succeed,
HookBehavior::Succeed,
)))
.build()
.expect("lifecycle should build");
let timeouts = LifecycleTimeouts::new(
Duration::from_millis(5),
Duration::from_millis(50),
Duration::from_millis(50),
)
.expect("timeouts should be valid");
let error = lifecycle
.run_with_timeouts(LifecycleStage::AfterShutdown, timeouts)
.await
.expect_err("hung shutdown hook should be reported");
assert_eq!(error.failures().len(), 1);
assert_eq!(
error.failures()[0].kind(),
LifecycleFailureKind::HookTimedOut
);
assert_eq!(
&*seen.lock().unwrap(),
&["run:third", "run:hung", "run:first"]
);
}
#[tokio::test]
async fn lifecycle_deadline_cancels_a_hung_hook() {
let seen = Arc::new(Mutex::new(Vec::new()));
let lifecycle = Lifecycle::builder()
.hook(Arc::new(ControlledHook::new(
"hung",
Arc::clone(&seen),
HookBehavior::Hang,
HookBehavior::Succeed,
)))
.build()
.expect("lifecycle should build");
let timeouts = LifecycleTimeouts::new(
Duration::from_millis(5),
Duration::from_millis(50),
Duration::from_millis(50),
)
.expect("timeouts should be valid");
let error = time::timeout(
Duration::from_millis(100),
lifecycle.run_with_timeouts(LifecycleStage::BeforeStart, timeouts),
)
.await
.expect("outer timeout should not elapse")
.expect_err("hung hook should time out");
assert!(error.timed_out());
assert_eq!(
error.failures()[0].kind(),
LifecycleFailureKind::HookTimedOut
);
assert_eq!(error.error_kind(), ErrorKind::Timeout);
assert_eq!(&*seen.lock().unwrap(), &["run:hung", "rollback:hung"]);
}
#[tokio::test]
async fn lifecycle_stage_deadline_bounds_the_whole_process() {
let seen = Arc::new(Mutex::new(Vec::new()));
let lifecycle = Lifecycle::builder()
.hook(Arc::new(ControlledHook::new(
"first",
Arc::clone(&seen),
HookBehavior::Delay(Duration::from_millis(30)),
HookBehavior::Succeed,
)))
.hook(Arc::new(ControlledHook::new(
"hung",
Arc::clone(&seen),
HookBehavior::Hang,
HookBehavior::Succeed,
)))
.build()
.expect("lifecycle should build");
let timeouts = LifecycleTimeouts::new(
Duration::from_millis(40),
Duration::from_millis(60),
Duration::from_millis(80),
)
.expect("timeouts should be valid");
let error = time::timeout(
Duration::from_millis(200),
lifecycle.run_with_timeouts(LifecycleStage::AfterStart, timeouts),
)
.await
.expect("whole stage must remain bounded")
.expect_err("global stage deadline should elapse");
assert_eq!(
error.failures()[0].kind(),
LifecycleFailureKind::StageTimedOut
);
assert!(error.cleanup_failures().is_empty());
assert_eq!(
&*seen.lock().unwrap(),
&["run:first", "run:hung", "rollback:hung", "rollback:first"]
);
}
#[tokio::test(start_paused = true)]
async fn expired_startup_stage_does_not_roll_back_an_unstarted_hook() {
let seen = Arc::new(Mutex::new(Vec::new()));
let lifecycle = Lifecycle::builder()
.hook(Arc::new(ControlledHook::new(
"first",
seen.clone(),
HookBehavior::Delay(Duration::from_secs(1)),
HookBehavior::Succeed,
)))
.hook(Arc::new(ControlledHook::new(
"unstarted",
seen.clone(),
HookBehavior::Succeed,
HookBehavior::Succeed,
)))
.build()
.unwrap();
let timeouts = LifecycleTimeouts::new(
Duration::from_secs(1),
Duration::from_secs(1),
Duration::from_secs(1),
)
.unwrap();
let error = lifecycle
.run_with_timeouts(LifecycleStage::BeforeStart, timeouts)
.await
.expect_err("the first hook consumes the stage budget");
assert_eq!(error.failures()[0].hook(), Some("unstarted"));
assert_eq!(
error.failures()[0].kind(),
LifecycleFailureKind::StageTimedOut
);
assert!(error.cleanup_failures().is_empty());
assert_eq!(&*seen.lock().unwrap(), &["run:first", "rollback:first"]);
}
#[tokio::test]
async fn cleanup_has_an_independent_bounded_budget() {
let seen = Arc::new(Mutex::new(Vec::new()));
let lifecycle = Lifecycle::builder()
.hook(Arc::new(ControlledHook::new(
"first",
Arc::clone(&seen),
HookBehavior::Succeed,
HookBehavior::Delay(Duration::from_millis(40)),
)))
.hook(Arc::new(ControlledHook::new(
"second",
Arc::clone(&seen),
HookBehavior::Fail,
HookBehavior::Delay(Duration::from_millis(40)),
)))
.build()
.expect("lifecycle should build");
let timeouts = LifecycleTimeouts::new(
Duration::from_millis(50),
Duration::from_millis(50),
Duration::from_millis(50),
)
.expect("timeouts should be valid");
let error = time::timeout(
Duration::from_millis(150),
lifecycle.run_with_timeouts(LifecycleStage::BeforeStart, timeouts),
)
.await
.expect("independent cleanup budget must remain bounded")
.expect_err("startup should fail");
assert_eq!(error.failures().len(), 1);
assert_eq!(error.cleanup_failures().len(), 1);
assert_eq!(error.cleanup_failures()[0].hook(), Some("first"));
assert_eq!(
error.cleanup_failures()[0].kind(),
LifecycleFailureKind::CleanupTimedOut
);
assert!(error.timed_out());
assert_eq!(
&*seen.lock().unwrap(),
&[
"run:first",
"run:second",
"rollback:second",
"rollback:first"
]
);
}
#[tokio::test]
async fn stateful_startup_rolls_back_both_stages_after_after_start_failure() {
let seen = Arc::new(Mutex::new(Vec::new()));
let lifecycle = Lifecycle::builder()
.hook(Arc::new(StageAwareHook::new(
"first",
Arc::clone(&seen),
false,
)))
.hook(Arc::new(StageAwareHook::new(
"second",
Arc::clone(&seen),
true,
)))
.build()
.expect("lifecycle should build");
let mut startup = lifecycle.startup();
assert_eq!(startup.next_stage(), Some(LifecycleStage::BeforeStart));
let before = startup
.run_next()
.await
.expect("before-start should succeed")
.expect("before-start stage should be present");
assert_eq!(before.stage(), LifecycleStage::BeforeStart);
assert_eq!(startup.next_stage(), Some(LifecycleStage::AfterStart));
let error = startup
.run_next()
.await
.expect_err("after-start should fail");
assert_eq!(error.stage(), LifecycleStage::AfterStart);
assert!(error.cleanup_failures().is_empty());
assert_eq!(startup.next_stage(), None);
assert_eq!(
&*seen.lock().unwrap(),
&[
"run:first:before_start",
"run:second:before_start",
"run:first:after_start",
"run:second:after_start",
"rollback:second:after_start",
"rollback:first:after_start",
"rollback:second:before_start",
"rollback:first:before_start",
]
);
}
#[tokio::test]
async fn stateful_startup_can_abort_between_stages_in_reverse_order() {
let seen = Arc::new(Mutex::new(Vec::new()));
let lifecycle = Lifecycle::builder()
.hook(Arc::new(StageAwareHook::new(
"first",
Arc::clone(&seen),
false,
)))
.hook(Arc::new(StageAwareHook::new(
"second",
Arc::clone(&seen),
false,
)))
.build()
.expect("lifecycle should build");
let mut startup = lifecycle.startup();
startup
.run_next()
.await
.expect("before-start should succeed");
startup.abort().await.expect("abort should clean up");
startup.abort().await.expect("abort should be idempotent");
assert_eq!(startup.next_stage(), None);
assert_eq!(
&*seen.lock().unwrap(),
&[
"run:first:before_start",
"run:second:before_start",
"rollback:second:before_start",
"rollback:first:before_start",
]
);
}
#[tokio::test]
async fn stateful_startup_abort_retains_a_cancelled_current_candidate() {
let seen = Arc::new(Mutex::new(Vec::new()));
let lifecycle = Lifecycle::builder()
.hook(Arc::new(ControlledHook::new(
"partial",
Arc::clone(&seen),
HookBehavior::Hang,
HookBehavior::Succeed,
)))
.build()
.expect("lifecycle should build");
let mut startup = lifecycle.startup();
time::timeout(Duration::from_millis(5), startup.run_next())
.await
.expect_err("outer cancellation should win before lifecycle timeouts");
assert_eq!(startup.next_stage(), None);
assert!(startup.run_next().await.unwrap().is_none());
startup
.abort()
.await
.expect("abort should roll back the cancelled candidate");
assert_eq!(&*seen.lock().unwrap(), &["run:partial", "rollback:partial"]);
}
#[tokio::test(start_paused = true)]
async fn retrying_cancelled_cleanup_does_not_repeat_successful_rollbacks() {
let seen = Arc::new(Mutex::new(Vec::new()));
let lifecycle = Lifecycle::builder()
.hook(Arc::new(ControlledHook::new(
"first",
seen.clone(),
HookBehavior::Succeed,
HookBehavior::Delay(Duration::from_secs(1)),
)))
.hook(Arc::new(ControlledHook::new(
"second",
seen.clone(),
HookBehavior::Succeed,
HookBehavior::Succeed,
)))
.build()
.unwrap();
let mut startup = lifecycle.startup();
startup.run_next().await.unwrap();
assert!(
time::timeout(Duration::from_millis(5), startup.abort())
.await
.is_err()
);
startup.abort().await.unwrap();
startup.abort().await.unwrap();
assert_eq!(
&*seen.lock().unwrap(),
&[
"run:first",
"run:second",
"rollback:second",
"rollback:first",
"rollback:first"
]
);
}
#[tokio::test]
async fn cleanup_retries_only_failed_actions_in_reverse_order() {
let seen = Arc::new(Mutex::new(Vec::new()));
let lifecycle = Lifecycle::builder()
.hook(Arc::new(ControlledHook::new(
"first",
seen.clone(),
HookBehavior::Succeed,
HookBehavior::Fail,
)))
.hook(Arc::new(ControlledHook::new(
"second",
seen.clone(),
HookBehavior::Succeed,
HookBehavior::Succeed,
)))
.hook(Arc::new(ControlledHook::new(
"third",
seen.clone(),
HookBehavior::Succeed,
HookBehavior::Fail,
)))
.build()
.unwrap();
let mut startup = lifecycle.startup();
startup.run_next().await.unwrap();
for _ in 0..2 {
let error = startup.abort().await.unwrap_err();
assert_eq!(
error
.cleanup_failures()
.iter()
.map(LifecycleFailure::hook)
.collect::<Vec<_>>(),
[Some("third"), Some("first")]
);
}
assert_eq!(
&*seen.lock().unwrap(),
&[
"run:first",
"run:second",
"run:third",
"rollback:third",
"rollback:second",
"rollback:first",
"rollback:third",
"rollback:first"
]
);
}
#[test]
fn lifecycle_timeouts_reject_invalid_bounds() {
assert!(matches!(
LifecycleTimeouts::new(
Duration::ZERO,
Duration::from_secs(1),
Duration::from_secs(1)
),
Err(LifecycleTimeoutError::InvalidHookTimeout)
));
assert!(matches!(
LifecycleTimeouts::new(
Duration::from_secs(2),
Duration::from_secs(1),
Duration::from_secs(2)
),
Err(LifecycleTimeoutError::HookExceedsStage)
));
assert!(matches!(
LifecycleTimeouts::new(
Duration::from_secs(1),
Duration::from_secs(2),
Duration::ZERO
),
Err(LifecycleTimeoutError::InvalidCleanupTimeout)
));
assert!(matches!(
LifecycleTimeouts::new(
Duration::from_secs(2),
Duration::from_secs(2),
Duration::from_secs(1)
),
Err(LifecycleTimeoutError::HookExceedsCleanup)
));
assert_eq!(
Lifecycle::default().timeouts(),
LifecycleTimeouts::default()
);
}
struct RecordingHook {
name: &'static str,
seen: Arc<Mutex<Vec<&'static str>>>,
}
impl RecordingHook {
fn new(name: &'static str, seen: Arc<Mutex<Vec<&'static str>>>) -> Self {
Self { name, seen }
}
}
#[async_trait]
impl LifecycleHook for RecordingHook {
fn name(&self) -> &'static str {
self.name
}
async fn run(&self, _stage: LifecycleStage) -> Result<(), BoxError> {
self.seen.lock().unwrap().push(self.name);
Ok(())
}
}
struct FailingHook;
#[async_trait]
impl LifecycleHook for FailingHook {
fn name(&self) -> &'static str {
"failing"
}
async fn run(&self, _stage: LifecycleStage) -> Result<(), BoxError> {
Err(std::io::Error::other("private failure").into())
}
}
#[derive(Clone, Copy)]
enum HookBehavior {
Succeed,
Fail,
Hang,
Delay(Duration),
}
struct ControlledHook {
name: &'static str,
seen: Arc<Mutex<Vec<String>>>,
run_behavior: HookBehavior,
rollback_behavior: HookBehavior,
}
impl ControlledHook {
fn new(
name: &'static str,
seen: Arc<Mutex<Vec<String>>>,
run_behavior: HookBehavior,
rollback_behavior: HookBehavior,
) -> Self {
Self {
name,
seen,
run_behavior,
rollback_behavior,
}
}
async fn apply(
&self,
operation: &'static str,
behavior: HookBehavior,
) -> Result<(), BoxError> {
self.seen
.lock()
.unwrap()
.push(format!("{operation}:{}", self.name));
match behavior {
HookBehavior::Succeed => Ok(()),
HookBehavior::Fail if operation == "rollback" => {
Err(std::io::Error::other("private rollback failure").into())
}
HookBehavior::Fail => Err(std::io::Error::other("private run failure").into()),
HookBehavior::Hang => std::future::pending().await,
HookBehavior::Delay(duration) => {
time::sleep(duration).await;
Ok(())
}
}
}
}
#[async_trait]
impl LifecycleHook for ControlledHook {
fn name(&self) -> &'static str {
self.name
}
async fn run(&self, _stage: LifecycleStage) -> Result<(), BoxError> {
self.apply("run", self.run_behavior).await
}
async fn rollback(&self, _startup_stage: LifecycleStage) -> Result<(), BoxError> {
self.apply("rollback", self.rollback_behavior).await
}
}
struct StageAwareHook {
name: &'static str,
seen: Arc<Mutex<Vec<String>>>,
fail_after_start: bool,
}
impl StageAwareHook {
fn new(name: &'static str, seen: Arc<Mutex<Vec<String>>>, fail_after_start: bool) -> Self {
Self {
name,
seen,
fail_after_start,
}
}
fn record(&self, operation: &str, stage: LifecycleStage) {
self.seen
.lock()
.unwrap()
.push(format!("{operation}:{}:{}", self.name, stage.as_str()));
}
}
#[async_trait]
impl LifecycleHook for StageAwareHook {
fn name(&self) -> &'static str {
self.name
}
async fn run(&self, stage: LifecycleStage) -> Result<(), BoxError> {
self.record("run", stage);
if self.fail_after_start && stage == LifecycleStage::AfterStart {
return Err(std::io::Error::other("private after-start failure").into());
}
Ok(())
}
async fn rollback(&self, startup_stage: LifecycleStage) -> Result<(), BoxError> {
self.record("rollback", startup_stage);
Ok(())
}
}
}