mod signals;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::broadcast;
pub use signals::{setup_default_handlers, wait_for_ctrl_c, Signal, SignalHandler};
pub struct ShutdownCoordinator {
shutdown: Arc<AtomicBool>,
in_flight: Arc<AtomicU64>,
notify: broadcast::Sender<()>,
drain_timeout: Duration,
initiated: AtomicBool,
}
impl ShutdownCoordinator {
pub fn new(drain_timeout: Duration) -> Self {
let (notify, _) = broadcast::channel(16);
Self {
shutdown: Arc::new(AtomicBool::new(false)),
in_flight: Arc::new(AtomicU64::new(0)),
notify,
drain_timeout,
initiated: AtomicBool::new(false),
}
}
pub fn with_default_timeout() -> Self {
Self::new(Duration::from_secs(30))
}
pub fn shutdown(&self) {
if self.initiated.swap(true, Ordering::SeqCst) {
return;
}
self.shutdown.store(true, Ordering::SeqCst);
let _ = self.notify.send(());
}
pub fn is_shutdown(&self) -> bool {
self.shutdown.load(Ordering::SeqCst)
}
pub fn in_flight_count(&self) -> u64 {
self.in_flight.load(Ordering::Acquire)
}
pub async fn wait_drain(&self) -> bool {
let deadline = Instant::now() + self.drain_timeout;
while self.in_flight.load(Ordering::Acquire) > 0 {
if Instant::now() > deadline {
return false;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
true
}
pub async fn wait_drain_with_timeout(&self, timeout: Duration) -> bool {
let deadline = Instant::now() + timeout;
while self.in_flight.load(Ordering::Acquire) > 0 {
if Instant::now() > deadline {
return false;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
true
}
pub fn request_start(&self) -> RequestGuard {
if self.is_shutdown() {
return RequestGuard { counter: None };
}
self.in_flight.fetch_add(1, Ordering::AcqRel);
if self.is_shutdown() {
self.decrement_in_flight();
return RequestGuard { counter: None };
}
RequestGuard {
counter: Some(Arc::clone(&self.in_flight)),
}
}
pub fn try_request_start(&self) -> Option<RequestGuard> {
let guard = self.request_start();
guard.is_active().then_some(guard)
}
pub fn increment_in_flight(&self) {
if self.is_shutdown() {
return;
}
self.in_flight.fetch_add(1, Ordering::AcqRel);
if self.is_shutdown() {
self.decrement_in_flight();
}
}
pub fn decrement_in_flight(&self) {
let _ = self
.in_flight
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| {
value.checked_sub(1)
});
}
pub fn subscribe(&self) -> broadcast::Receiver<()> {
self.notify.subscribe()
}
pub fn drain_timeout(&self) -> Duration {
self.drain_timeout
}
pub fn set_drain_timeout(&mut self, timeout: Duration) {
self.drain_timeout = timeout;
}
pub fn into_arc(self) -> Arc<Self> {
Arc::new(self)
}
}
impl Default for ShutdownCoordinator {
fn default() -> Self {
Self::with_default_timeout()
}
}
pub struct RequestGuard {
counter: Option<Arc<AtomicU64>>,
}
impl RequestGuard {
pub fn is_active(&self) -> bool {
self.counter.is_some()
}
}
impl Drop for RequestGuard {
fn drop(&mut self) {
if let Some(counter) = &self.counter {
let _ = counter.fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| {
value.checked_sub(1)
});
}
}
}
#[derive(Debug, Clone)]
pub struct ShutdownProgress {
pub initiated: bool,
pub in_flight: u64,
pub elapsed: Option<Duration>,
pub timeout: Duration,
}
impl ShutdownProgress {
pub fn to_log_message(&self) -> String {
if !self.initiated {
return "Shutdown not initiated".to_string();
}
let elapsed_str = self
.elapsed
.map(|d| format!("{:.1}s", d.as_secs_f64()))
.unwrap_or_else(|| "unknown".to_string());
let timeout_str = format!("{:.1}s", self.timeout.as_secs_f64());
if self.in_flight == 0 {
format!("Shutdown complete after {}", elapsed_str)
} else {
format!(
"Shutdown in progress: {} in-flight requests, elapsed: {}, timeout: {}",
self.in_flight, elapsed_str, timeout_str
)
}
}
pub fn is_complete(&self) -> bool {
self.initiated && self.in_flight == 0
}
pub fn is_timed_out(&self) -> bool {
if let Some(elapsed) = self.elapsed {
elapsed >= self.timeout
} else {
false
}
}
}
pub struct ShutdownLogger {
coordinator: Arc<ShutdownCoordinator>,
start_time: std::sync::Mutex<Option<Instant>>,
log_interval: Duration,
}
impl ShutdownLogger {
pub fn new(coordinator: Arc<ShutdownCoordinator>, log_interval: Duration) -> Self {
Self {
coordinator,
start_time: std::sync::Mutex::new(None),
log_interval,
}
}
pub fn with_default_interval(coordinator: Arc<ShutdownCoordinator>) -> Self {
Self::new(coordinator, Duration::from_secs(1))
}
pub fn start(&self) {
let mut start_time = self.start_time.lock().unwrap();
if start_time.is_none() {
*start_time = Some(Instant::now());
}
}
pub fn progress(&self) -> ShutdownProgress {
let elapsed = self.start_time.lock().unwrap().map(|t| t.elapsed());
ShutdownProgress {
initiated: self.coordinator.is_shutdown(),
in_flight: self.coordinator.in_flight_count(),
elapsed,
timeout: self.coordinator.drain_timeout(),
}
}
pub fn log_progress(&self) {
let progress = self.progress();
eprintln!("[SHUTDOWN] {}", progress.to_log_message());
}
pub async fn run_logging(&self) {
self.start();
loop {
let progress = self.progress();
eprintln!("[SHUTDOWN] {}", progress.to_log_message());
if progress.is_complete() {
eprintln!("[SHUTDOWN] Graceful shutdown complete");
break;
}
if progress.is_timed_out() {
eprintln!(
"[SHUTDOWN] Timeout reached with {} requests still in-flight",
progress.in_flight
);
break;
}
tokio::time::sleep(self.log_interval).await;
}
}
pub fn log_interval(&self) -> Duration {
self.log_interval
}
}
impl ShutdownCoordinator {
pub fn progress(&self) -> ShutdownProgress {
ShutdownProgress {
initiated: self.is_shutdown(),
in_flight: self.in_flight_count(),
elapsed: None, timeout: self.drain_timeout,
}
}
pub fn create_logger(self: &Arc<Self>) -> ShutdownLogger {
ShutdownLogger::with_default_interval(Arc::clone(self))
}
pub fn create_logger_with_interval(self: &Arc<Self>, interval: Duration) -> ShutdownLogger {
ShutdownLogger::new(Arc::clone(self), interval)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_shutdown_coordinator_creation() {
let coord = ShutdownCoordinator::new(Duration::from_secs(10));
assert!(!coord.is_shutdown());
assert_eq!(coord.in_flight_count(), 0);
}
#[test]
fn test_shutdown_signal() {
let coord = ShutdownCoordinator::default();
assert!(!coord.is_shutdown());
coord.shutdown();
assert!(coord.is_shutdown());
coord.shutdown();
assert!(coord.is_shutdown());
}
#[test]
fn test_request_guard() {
let coord = ShutdownCoordinator::default();
assert_eq!(coord.in_flight_count(), 0);
{
let _guard1 = coord.request_start();
assert_eq!(coord.in_flight_count(), 1);
{
let _guard2 = coord.request_start();
assert_eq!(coord.in_flight_count(), 2);
}
assert_eq!(coord.in_flight_count(), 1);
}
assert_eq!(coord.in_flight_count(), 0);
}
#[test]
fn test_manual_increment_decrement() {
let coord = ShutdownCoordinator::default();
coord.increment_in_flight();
coord.increment_in_flight();
assert_eq!(coord.in_flight_count(), 2);
coord.decrement_in_flight();
assert_eq!(coord.in_flight_count(), 1);
coord.decrement_in_flight();
assert_eq!(coord.in_flight_count(), 0);
}
#[tokio::test]
async fn test_wait_drain_immediate() {
let coord = ShutdownCoordinator::new(Duration::from_millis(100));
let result = coord.wait_drain().await;
assert!(result);
}
#[tokio::test]
async fn test_wait_drain_with_requests() {
let coord = Arc::new(ShutdownCoordinator::new(Duration::from_secs(1)));
let coord_clone = Arc::clone(&coord);
let guard = coord.request_start();
tokio::spawn(async move {
tokio::time::sleep(Duration::from_millis(50)).await;
drop(guard);
});
let result = coord_clone.wait_drain().await;
assert!(result);
assert_eq!(coord_clone.in_flight_count(), 0);
}
#[tokio::test]
async fn test_wait_drain_timeout() {
let coord = ShutdownCoordinator::new(Duration::from_millis(50));
let _guard = coord.request_start();
let result = coord.wait_drain().await;
assert!(!result);
assert_eq!(coord.in_flight_count(), 1);
}
#[tokio::test]
async fn test_shutdown_notification() {
let coord = ShutdownCoordinator::default();
let mut rx = coord.subscribe();
let coord_clone = Arc::new(coord);
let coord_for_shutdown = Arc::clone(&coord_clone);
tokio::spawn(async move {
tokio::time::sleep(Duration::from_millis(10)).await;
coord_for_shutdown.shutdown();
});
let result = tokio::time::timeout(Duration::from_millis(100), rx.recv()).await;
assert!(result.is_ok());
}
#[test]
fn test_shutdown_progress() {
let coord = ShutdownCoordinator::new(Duration::from_secs(30));
let progress = coord.progress();
assert!(!progress.initiated);
assert_eq!(progress.in_flight, 0);
assert_eq!(progress.timeout, Duration::from_secs(30));
coord.increment_in_flight();
coord.shutdown();
let progress = coord.progress();
assert!(progress.initiated);
assert_eq!(progress.in_flight, 1);
}
}