#[cfg(feature = "async")]
use std::future::Future;
#[cfg(feature = "async")]
use std::pin::Pin;
use std::time::{Duration, Instant};
use std::cell::RefCell;
#[cfg(feature = "async")]
use std::sync::{Arc, RwLock};
use crate::error::TraitKitError;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum ShutdownPhase {
StopRequests,
DrainQueue,
CloseConnections,
}
impl ShutdownPhase {
#[must_use]
pub fn all_phases() -> &'static [ShutdownPhase] {
&[
ShutdownPhase::StopRequests,
ShutdownPhase::DrainQueue,
ShutdownPhase::CloseConnections,
]
}
#[must_use]
pub const fn as_str(&self) -> &'static str {
match self {
Self::StopRequests => "stop_requests",
Self::DrainQueue => "drain_queue",
Self::CloseConnections => "close_connections",
}
}
}
type SyncShutdownHook = Box<dyn FnOnce()>;
struct PhaseConfig {
hooks: Vec<SyncShutdownHook>,
timeout: Duration,
}
impl PhaseConfig {
fn new(timeout: Duration) -> Self {
Self {
hooks: Vec::new(),
timeout,
}
}
}
pub struct ShutdownCoordinator {
phases: RefCell<[PhaseConfig; 3]>,
global_timeout: RefCell<Option<Duration>>,
}
impl ShutdownCoordinator {
#[must_use]
pub fn new() -> Self {
const DEFAULT_PHASE_TIMEOUT: Duration = Duration::from_secs(30);
Self {
phases: RefCell::new([
PhaseConfig::new(DEFAULT_PHASE_TIMEOUT),
PhaseConfig::new(DEFAULT_PHASE_TIMEOUT),
PhaseConfig::new(DEFAULT_PHASE_TIMEOUT),
]),
global_timeout: RefCell::new(None),
}
}
pub fn set_global_timeout(&self, timeout: Duration) {
*self.global_timeout.borrow_mut() = Some(timeout);
}
pub fn set_phase_timeout(&self, phase: ShutdownPhase, timeout: Duration) {
let idx = Self::phase_index(phase);
self.phases.borrow_mut()[idx].timeout = timeout;
}
pub fn register_hook<F>(&self, phase: ShutdownPhase, hook: F)
where
F: FnOnce() + 'static,
{
let idx = Self::phase_index(phase);
self.phases.borrow_mut()[idx].hooks.push(Box::new(hook));
}
#[must_use = "shutdown returns phase results; ignoring it may hide timeout events"]
pub fn shutdown(&self) -> Vec<ShutdownPhaseResult> {
let global_start = Instant::now();
let global_timeout = *self.global_timeout.borrow();
let mut results = Vec::with_capacity(3);
for phase in ShutdownPhase::all_phases() {
if let Some(gt) = global_timeout
&& global_start.elapsed() >= gt
{
results.push(ShutdownPhaseResult {
phase: *phase,
timed_out: true,
elapsed: global_start.elapsed(),
});
continue;
}
let result = self.execute_phase(*phase);
results.push(result);
}
results
}
fn execute_phase(&self, phase: ShutdownPhase) -> ShutdownPhaseResult {
let idx = Self::phase_index(phase);
let start = Instant::now();
let hooks: Vec<SyncShutdownHook> = {
let mut phases = self.phases.borrow_mut();
std::mem::take(&mut phases[idx].hooks)
};
let timeout = self.phases.borrow()[idx].timeout;
for hook in hooks {
if start.elapsed() >= timeout {
return ShutdownPhaseResult {
phase,
timed_out: true,
elapsed: start.elapsed(),
};
}
hook();
}
ShutdownPhaseResult {
phase,
timed_out: false,
elapsed: start.elapsed(),
}
}
const fn phase_index(phase: ShutdownPhase) -> usize {
match phase {
ShutdownPhase::StopRequests => 0,
ShutdownPhase::DrainQueue => 1,
ShutdownPhase::CloseConnections => 2,
}
}
}
impl Default for ShutdownCoordinator {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct ShutdownPhaseResult {
pub phase: ShutdownPhase,
pub timed_out: bool,
pub elapsed: Duration,
}
impl ShutdownPhaseResult {
#[must_use]
pub fn is_ok(&self) -> bool {
!self.timed_out
}
}
#[derive(Debug, Clone)]
pub struct ShutdownResult {
pub phases: Vec<ShutdownPhaseResult>,
}
impl ShutdownResult {
#[must_use]
pub fn is_ok(&self) -> bool {
self.phases.iter().all(ShutdownPhaseResult::is_ok)
}
#[must_use]
pub fn timed_out_phases(&self) -> Vec<ShutdownPhase> {
self.phases
.iter()
.filter(|r| r.timed_out)
.map(|r| r.phase)
.collect()
}
pub fn into_result(self) -> Result<Self, TraitKitError> {
if self.is_ok() {
Ok(self)
} else {
Err(TraitKitError::ShutdownTimedOut {
phases: self.timed_out_phases(),
})
}
}
}
#[cfg(feature = "async")]
type AsyncShutdownHook =
Box<dyn FnOnce() -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
#[cfg(feature = "async")]
struct AsyncPhaseConfig {
hooks: Vec<AsyncShutdownHook>,
timeout: Duration,
}
#[cfg(feature = "async")]
impl AsyncPhaseConfig {
fn new(timeout: Duration) -> Self {
Self {
hooks: Vec::new(),
timeout,
}
}
}
#[cfg(feature = "async")]
pub struct AsyncShutdownCoordinator {
phases: Arc<RwLock<[AsyncPhaseConfig; 3]>>,
global_timeout: Arc<RwLock<Option<Duration>>>,
}
#[cfg(feature = "async")]
impl AsyncShutdownCoordinator {
#[must_use]
pub fn new() -> Self {
const DEFAULT_PHASE_TIMEOUT: Duration = Duration::from_secs(30);
Self {
phases: Arc::new(RwLock::new([
AsyncPhaseConfig::new(DEFAULT_PHASE_TIMEOUT),
AsyncPhaseConfig::new(DEFAULT_PHASE_TIMEOUT),
AsyncPhaseConfig::new(DEFAULT_PHASE_TIMEOUT),
])),
global_timeout: Arc::new(RwLock::new(None)),
}
}
pub fn set_global_timeout(&self, timeout: Duration) {
*self.global_timeout.write().expect("lock poisoned") = Some(timeout);
}
pub fn set_phase_timeout(&self, phase: ShutdownPhase, timeout: Duration) {
let idx = Self::phase_index(phase);
self.phases.write().expect("lock poisoned")[idx].timeout = timeout;
}
pub fn register_hook<F, Fut>(&self, phase: ShutdownPhase, hook: F) -> Result<(), TraitKitError>
where
F: FnOnce() -> Pin<Box<Fut>> + Send + Sync + 'static,
Fut: Future<Output = ()> + Send + 'static,
{
let idx = Self::phase_index(phase);
let boxed: AsyncShutdownHook =
Box::new(move || -> Pin<Box<dyn Future<Output = ()> + Send>> {
let fut = hook();
Box::pin(fut) as Pin<Box<dyn Future<Output = ()> + Send>>
});
self.phases
.write()
.map_err(|_| TraitKitError::BuildFailed {
context: format!("shutdown phase `{}`", phase.as_str()),
source: Box::new(std::io::Error::other("RwLock poisoned")),
})?
.get_mut(idx)
.expect("index in range")
.hooks
.push(boxed);
Ok(())
}
#[must_use = "shutdown returns phase result; ignoring it may hide timeout events"]
pub async fn shutdown(&self) -> ShutdownResult {
let global_start = Instant::now();
let global_timeout = *self.global_timeout.read().expect("lock poisoned");
let mut results = Vec::with_capacity(3);
for phase in ShutdownPhase::all_phases() {
if let Some(gt) = global_timeout
&& global_start.elapsed() >= gt
{
results.push(ShutdownPhaseResult {
phase: *phase,
timed_out: true,
elapsed: global_start.elapsed(),
});
continue;
}
let result = self.execute_phase(*phase).await;
results.push(result);
}
ShutdownResult { phases: results }
}
async fn execute_phase(&self, phase: ShutdownPhase) -> ShutdownPhaseResult {
let idx = Self::phase_index(phase);
let start = Instant::now();
let hooks: Vec<AsyncShutdownHook> = {
let mut phases = self.phases.write().expect("lock poisoned");
std::mem::take(&mut phases[idx].hooks)
};
let timeout = self.phases.read().expect("lock poisoned")[idx].timeout;
for hook in hooks {
if start.elapsed() >= timeout {
return ShutdownPhaseResult {
phase,
timed_out: true,
elapsed: start.elapsed(),
};
}
hook().await;
}
ShutdownPhaseResult {
phase,
timed_out: false,
elapsed: start.elapsed(),
}
}
const fn phase_index(phase: ShutdownPhase) -> usize {
match phase {
ShutdownPhase::StopRequests => 0,
ShutdownPhase::DrainQueue => 1,
ShutdownPhase::CloseConnections => 2,
}
}
}
#[cfg(feature = "async")]
impl Default for AsyncShutdownCoordinator {
fn default() -> Self {
Self::new()
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicUsize, Ordering};
#[test]
fn shutdown_phase_all_phases_returns_three() {
assert_eq!(ShutdownPhase::all_phases().len(), 3);
}
#[test]
fn shutdown_phase_as_str_returns_readable_name() {
assert_eq!(ShutdownPhase::StopRequests.as_str(), "stop_requests");
assert_eq!(ShutdownPhase::DrainQueue.as_str(), "drain_queue");
assert_eq!(
ShutdownPhase::CloseConnections.as_str(),
"close_connections"
);
}
#[test]
fn shutdown_coordinator_executes_hooks_in_order() {
static ORDER: AtomicUsize = AtomicUsize::new(0);
let coord = ShutdownCoordinator::new();
coord.register_hook(ShutdownPhase::StopRequests, || {
assert_eq!(ORDER.fetch_add(1, Ordering::SeqCst), 0);
});
coord.register_hook(ShutdownPhase::StopRequests, || {
assert_eq!(ORDER.fetch_add(1, Ordering::SeqCst), 1);
});
coord.register_hook(ShutdownPhase::DrainQueue, || {
assert_eq!(ORDER.fetch_add(1, Ordering::SeqCst), 2);
});
coord.register_hook(ShutdownPhase::CloseConnections, || {
assert_eq!(ORDER.fetch_add(1, Ordering::SeqCst), 3);
});
let results = coord.shutdown();
assert_eq!(results.len(), 3);
assert!(results.iter().all(ShutdownPhaseResult::is_ok));
assert_eq!(ORDER.load(Ordering::SeqCst), 4);
}
#[test]
fn shutdown_coordinator_phase_order_is_correct() {
static PHASE_ORDER: std::sync::Mutex<Vec<ShutdownPhase>> =
std::sync::Mutex::new(Vec::new());
let coord = ShutdownCoordinator::new();
coord.register_hook(ShutdownPhase::CloseConnections, || {
PHASE_ORDER
.lock()
.unwrap()
.push(ShutdownPhase::CloseConnections);
});
coord.register_hook(ShutdownPhase::StopRequests, || {
PHASE_ORDER
.lock()
.unwrap()
.push(ShutdownPhase::StopRequests);
});
coord.register_hook(ShutdownPhase::DrainQueue, || {
PHASE_ORDER.lock().unwrap().push(ShutdownPhase::DrainQueue);
});
let _ = coord.shutdown();
let order = PHASE_ORDER.lock().unwrap();
assert_eq!(
*order,
vec![
ShutdownPhase::StopRequests,
ShutdownPhase::DrainQueue,
ShutdownPhase::CloseConnections,
]
);
}
#[test]
fn shutdown_coordinator_timeout_skips_remaining_hooks() {
static CALLED: AtomicUsize = AtomicUsize::new(0);
let coord = ShutdownCoordinator::new();
coord.set_phase_timeout(ShutdownPhase::DrainQueue, Duration::from_millis(10));
coord.register_hook(ShutdownPhase::StopRequests, || {
CALLED.fetch_add(1, Ordering::SeqCst);
});
coord.register_hook(ShutdownPhase::DrainQueue, || {
std::thread::sleep(Duration::from_millis(50));
CALLED.fetch_add(1, Ordering::SeqCst);
});
coord.register_hook(ShutdownPhase::DrainQueue, || {
CALLED.fetch_add(1, Ordering::SeqCst);
});
coord.register_hook(ShutdownPhase::CloseConnections, || {
CALLED.fetch_add(1, Ordering::SeqCst);
});
let results = coord.shutdown();
assert_eq!(results.len(), 3);
assert!(results[0].is_ok()); assert!(results[1].timed_out); assert!(results[2].is_ok());
assert_eq!(CALLED.load(Ordering::SeqCst), 3);
}
#[test]
fn shutdown_coordinator_global_timeout() {
static CALLED: AtomicUsize = AtomicUsize::new(0);
let coord = ShutdownCoordinator::new();
coord.set_global_timeout(Duration::from_millis(10));
coord.register_hook(ShutdownPhase::StopRequests, || {
std::thread::sleep(Duration::from_millis(50));
CALLED.fetch_add(1, Ordering::SeqCst);
});
coord.register_hook(ShutdownPhase::DrainQueue, || {
CALLED.fetch_add(1, Ordering::SeqCst);
});
coord.register_hook(ShutdownPhase::CloseConnections, || {
CALLED.fetch_add(1, Ordering::SeqCst);
});
let results = coord.shutdown();
assert_eq!(results.len(), 3);
assert_eq!(CALLED.load(Ordering::SeqCst), 1);
assert!(results[0].is_ok()); assert!(results[1].timed_out); assert!(results[2].timed_out); }
#[test]
fn shutdown_result_into_result_ok() {
let result = ShutdownResult {
phases: vec![
ShutdownPhaseResult {
phase: ShutdownPhase::StopRequests,
timed_out: false,
elapsed: Duration::from_millis(1),
},
ShutdownPhaseResult {
phase: ShutdownPhase::DrainQueue,
timed_out: false,
elapsed: Duration::from_millis(1),
},
ShutdownPhaseResult {
phase: ShutdownPhase::CloseConnections,
timed_out: false,
elapsed: Duration::from_millis(1),
},
],
};
assert!(result.is_ok());
assert!(result.timed_out_phases().is_empty());
assert!(result.into_result().is_ok());
}
#[test]
fn shutdown_result_into_result_timeout() {
let result = ShutdownResult {
phases: vec![
ShutdownPhaseResult {
phase: ShutdownPhase::StopRequests,
timed_out: false,
elapsed: Duration::from_millis(1),
},
ShutdownPhaseResult {
phase: ShutdownPhase::DrainQueue,
timed_out: true,
elapsed: Duration::from_secs(30),
},
ShutdownPhaseResult {
phase: ShutdownPhase::CloseConnections,
timed_out: false,
elapsed: Duration::from_millis(1),
},
],
};
assert!(!result.is_ok());
assert_eq!(result.timed_out_phases(), vec![ShutdownPhase::DrainQueue]);
let err = result.into_result().unwrap_err();
let msg = format!("{err}");
assert!(
msg.contains("drain_queue"),
"error should mention timed out phase: {msg}"
);
}
#[test]
fn shutdown_coordinator_default_works() {
let coord = ShutdownCoordinator::default();
let results = coord.shutdown();
assert_eq!(results.len(), 3);
assert!(results.iter().all(ShutdownPhaseResult::is_ok));
}
#[test]
fn shutdown_coordinator_empty_phases_succeed() {
let coord = ShutdownCoordinator::new();
let results = coord.shutdown();
assert_eq!(results.len(), 3);
for r in &results {
assert!(r.is_ok());
assert!(
r.elapsed.as_nanos() < 1_000_000,
"empty phase should be near-instant"
);
}
}
#[test]
fn shutdown_coordinator_hooks_not_reentrant() {
static CALL_COUNT: AtomicUsize = AtomicUsize::new(0);
let coord = ShutdownCoordinator::new();
coord.register_hook(ShutdownPhase::StopRequests, || {
CALL_COUNT.fetch_add(1, Ordering::SeqCst);
});
let _ = coord.shutdown();
assert_eq!(CALL_COUNT.load(Ordering::SeqCst), 1);
let _ = coord.shutdown();
assert_eq!(CALL_COUNT.load(Ordering::SeqCst), 1);
}
}
#[cfg(all(test, feature = "async"))]
mod async_tests {
use super::*;
use crate::test_helpers::block_on;
use std::sync::atomic::{AtomicUsize, Ordering};
#[test]
fn async_shutdown_coordinator_executes_hooks() {
static CALLED: AtomicUsize = AtomicUsize::new(0);
block_on(async {
let coord = AsyncShutdownCoordinator::new();
coord
.register_hook(ShutdownPhase::StopRequests, || {
Box::pin(async {
CALLED.fetch_add(1, Ordering::SeqCst);
})
})
.unwrap();
coord
.register_hook(ShutdownPhase::DrainQueue, || {
Box::pin(async {
CALLED.fetch_add(1, Ordering::SeqCst);
})
})
.unwrap();
coord
.register_hook(ShutdownPhase::CloseConnections, || {
Box::pin(async {
CALLED.fetch_add(1, Ordering::SeqCst);
})
})
.unwrap();
let result = coord.shutdown().await;
assert!(result.is_ok());
assert_eq!(CALLED.load(Ordering::SeqCst), 3);
});
}
#[test]
fn async_shutdown_coordinator_timeout() {
static CALLED: AtomicUsize = AtomicUsize::new(0);
block_on(async {
let coord = AsyncShutdownCoordinator::new();
coord.set_phase_timeout(ShutdownPhase::DrainQueue, Duration::from_millis(10));
coord
.register_hook(ShutdownPhase::StopRequests, || {
Box::pin(async {
CALLED.fetch_add(1, Ordering::SeqCst);
})
})
.unwrap();
coord
.register_hook(ShutdownPhase::DrainQueue, || {
Box::pin(async {
std::thread::sleep(Duration::from_millis(50));
CALLED.fetch_add(1, Ordering::SeqCst);
})
})
.unwrap();
coord
.register_hook(ShutdownPhase::DrainQueue, || {
Box::pin(async {
CALLED.fetch_add(1, Ordering::SeqCst);
})
})
.unwrap();
coord
.register_hook(ShutdownPhase::CloseConnections, || {
Box::pin(async {
CALLED.fetch_add(1, Ordering::SeqCst);
})
})
.unwrap();
let result = coord.shutdown().await;
assert!(!result.is_ok());
assert_eq!(result.timed_out_phases(), vec![ShutdownPhase::DrainQueue]);
assert_eq!(CALLED.load(Ordering::SeqCst), 3);
});
}
#[test]
fn async_shutdown_coordinator_default() {
block_on(async {
let coord = AsyncShutdownCoordinator::default();
let result = coord.shutdown().await;
assert!(result.is_ok());
});
}
#[test]
fn async_shutdown_coordinator_hooks_not_reentrant() {
static CALL_COUNT: AtomicUsize = AtomicUsize::new(0);
block_on(async {
let coord = AsyncShutdownCoordinator::new();
coord
.register_hook(ShutdownPhase::StopRequests, || {
Box::pin(async {
CALL_COUNT.fetch_add(1, Ordering::SeqCst);
})
})
.unwrap();
let _ = coord.shutdown().await;
assert_eq!(CALL_COUNT.load(Ordering::SeqCst), 1);
let _ = coord.shutdown().await;
assert_eq!(CALL_COUNT.load(Ordering::SeqCst), 1);
});
}
}