use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::Notify;
use crate::domain::repositories::task_repository::TaskRepository;
use log::{error, info};
use trait_kit::kit::shutdown::AsyncShutdownCoordinator;
pub use trait_kit::kit::shutdown::ShutdownPhase;
pub const DEFAULT_GRACEFUL_PERIOD_SECS: u64 = 30;
pub struct ShutdownCoordinator {
flag: AtomicBool,
notify: Arc<Notify>,
graceful_period: Duration,
phased: AsyncShutdownCoordinator,
}
impl ShutdownCoordinator {
pub fn new(graceful_period: Duration) -> Self {
let phased = AsyncShutdownCoordinator::new();
let _ = phased.set_global_timeout(graceful_period);
Self {
flag: AtomicBool::new(false),
notify: Arc::new(Notify::new()),
graceful_period,
phased,
}
}
pub fn register_shutdown_hook<F>(&self, phase: ShutdownPhase, hook: F)
where
F: FnOnce() -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>>
+ Send
+ Sync
+ 'static,
{
if let Err(e) = self.phased.register_hook(phase, hook) {
error!("Failed to register shutdown hook: {}", e);
}
}
pub async fn run_phased_shutdown(&self) -> bool {
match self.phased.shutdown().await {
Ok(result) => {
let timed_out = result.timed_out_phases();
if !timed_out.is_empty() {
let names = timed_out
.iter()
.map(|p| p.as_str())
.collect::<Vec<_>>()
.join(", ");
error!("Phased shutdown timed out in phase(s): {}", names);
}
let ok = result.is_ok();
info!("Phased shutdown complete (ok={}, phases executed)", ok);
ok
}
Err(e) => {
error!("Phased shutdown failed: {}", e);
false
}
}
}
pub fn with_defaults() -> Self {
Self::new(Duration::from_secs(DEFAULT_GRACEFUL_PERIOD_SECS))
}
pub fn trigger(&self) {
self.flag.store(true, Ordering::SeqCst);
self.notify.notify_waiters();
}
pub fn is_shutting_down(&self) -> bool {
self.flag.load(Ordering::SeqCst)
}
pub fn graceful_period(&self) -> Duration {
self.graceful_period
}
pub async fn wait_for_completion(&self) -> bool {
if self.is_shutting_down() {
return true;
}
self.notify.notified().await;
true
}
}
impl Default for ShutdownCoordinator {
fn default() -> Self {
Self::with_defaults()
}
}
#[cfg(unix)]
pub async fn listen_unix_signals(coordinator: Arc<ShutdownCoordinator>) -> std::io::Result<()> {
use log::info;
use tokio::signal::unix::{signal, SignalKind};
let mut sigterm = signal(SignalKind::terminate())?;
let mut sigint = signal(SignalKind::interrupt())?;
tokio::select! {
_ = sigterm.recv() => {
info!("SIGTERM received, initiating graceful shutdown");
}
_ = sigint.recv() => {
info!("SIGINT received, initiating graceful shutdown");
}
}
coordinator.trigger();
Ok(())
}
#[cfg(not(unix))]
pub async fn listen_unix_signals(coordinator: Arc<ShutdownCoordinator>) -> std::io::Result<()> {
use log::info;
tokio::signal::ctrl_c().await?;
info!("Ctrl+C received, initiating graceful shutdown");
coordinator.trigger();
Ok(())
}
pub async fn rollback_pending_tasks(
repository: &Arc<dyn TaskRepository>,
graceful_period: Duration,
worker_ids: &[crate::domain::id::Id],
) {
match tokio::time::timeout(
graceful_period,
repository.reset_stuck_tasks_for_workers(chrono::Duration::zero(), worker_ids),
)
.await
{
Ok(Ok(affected)) => {
info!(
"Rolled back {} in-flight tasks to queued during shutdown (workers: {})",
affected,
worker_ids.len()
);
}
Ok(Err(e)) => {
error!("Failed to roll back in-flight tasks during shutdown: {}", e);
}
Err(_) => {
error!(
"Roll back of in-flight tasks timed out after {:?}",
graceful_period
);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use async_trait::async_trait;
#[test]
fn test_initial_state_not_shutting_down() {
let coordinator = ShutdownCoordinator::with_defaults();
assert!(!coordinator.is_shutting_down());
}
#[test]
fn test_trigger_sets_shutting_down() {
let coordinator = ShutdownCoordinator::with_defaults();
coordinator.trigger();
assert!(coordinator.is_shutting_down());
}
#[test]
fn test_default_graceful_period_is_30s() {
let coordinator = ShutdownCoordinator::with_defaults();
assert_eq!(coordinator.graceful_period(), Duration::from_secs(30));
}
#[test]
fn test_custom_graceful_period() {
let coordinator = ShutdownCoordinator::new(Duration::from_millis(100));
assert_eq!(coordinator.graceful_period(), Duration::from_millis(100));
assert!(!coordinator.is_shutting_down());
}
#[tokio::test]
async fn test_trigger_before_wait_returns_immediately() {
let coordinator = Arc::new(ShutdownCoordinator::new(Duration::from_secs(60)));
coordinator.trigger();
let result = coordinator.wait_for_completion().await;
assert!(result);
}
#[tokio::test]
async fn test_wait_for_completion_unblocks_on_trigger() {
let coordinator = Arc::new(ShutdownCoordinator::new(Duration::from_secs(30)));
let coord_for_task = coordinator.clone();
tokio::spawn(async move {
tokio::time::sleep(Duration::from_millis(20)).await;
coord_for_task.trigger();
});
let start = std::time::Instant::now();
let result = coordinator.wait_for_completion().await;
let elapsed = start.elapsed();
assert!(result);
assert!(
elapsed < Duration::from_secs(5),
"should unblock promptly on trigger, got {:?}",
elapsed
);
}
#[tokio::test]
async fn test_wait_for_completion_waits_indefinitely_without_trigger() {
let coordinator = Arc::new(ShutdownCoordinator::new(Duration::from_millis(50)));
let wait_task = tokio::spawn({
let coordinator = coordinator.clone();
async move { coordinator.wait_for_completion().await }
});
tokio::time::sleep(Duration::from_millis(120)).await;
assert!(
!wait_task.is_finished(),
"wait_for_completion must not return without trigger \
(worker daemon would die after graceful_period)"
);
coordinator.trigger();
assert!(wait_task.await.unwrap(), "should return true after trigger");
}
#[tokio::test]
async fn test_phased_shutdown_runs_registered_hook() {
let coordinator = Arc::new(ShutdownCoordinator::new(Duration::from_secs(30)));
let counter = Arc::new(std::sync::atomic::AtomicU32::new(0));
let counter_in_hook = counter.clone();
coordinator.register_shutdown_hook(ShutdownPhase::DrainQueue, move || {
let counter = counter_in_hook.clone();
Box::pin(async move {
counter.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
})
});
assert!(coordinator.run_phased_shutdown().await);
assert_eq!(
counter.load(std::sync::atomic::Ordering::SeqCst),
1,
"registered DrainQueue hook must execute exactly once"
);
}
#[tokio::test]
async fn test_phased_shutdown_executes_phases_in_order() {
let coordinator = Arc::new(ShutdownCoordinator::new(Duration::from_secs(30)));
let order = Arc::new(parking_lot::Mutex::<Vec<&'static str>>::new(Vec::new()));
let o_close = order.clone();
coordinator.register_shutdown_hook(ShutdownPhase::CloseConnections, move || {
let o = o_close.clone();
Box::pin(async move { o.lock().push("close") })
});
let o_stop = order.clone();
coordinator.register_shutdown_hook(ShutdownPhase::StopRequests, move || {
let o = o_stop.clone();
Box::pin(async move { o.lock().push("stop") })
});
let o_drain = order.clone();
coordinator.register_shutdown_hook(ShutdownPhase::DrainQueue, move || {
let o = o_drain.clone();
Box::pin(async move { o.lock().push("drain") })
});
assert!(coordinator.run_phased_shutdown().await);
assert_eq!(
*order.lock(),
vec!["stop", "drain", "close"],
"hooks must run in phase order regardless of registration order"
);
}
#[tokio::test]
async fn test_phased_shutdown_skips_remaining_hooks_on_global_timeout() {
let coordinator = Arc::new(ShutdownCoordinator::new(Duration::from_millis(50)));
let slow_ran = Arc::new(std::sync::atomic::AtomicU32::new(0));
let skipped_ran = Arc::new(std::sync::atomic::AtomicU32::new(0));
let slow_flag = slow_ran.clone();
coordinator.register_shutdown_hook(ShutdownPhase::DrainQueue, move || {
let flag = slow_flag.clone();
Box::pin(async move {
tokio::time::sleep(Duration::from_millis(100)).await;
flag.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
})
});
let skipped_flag = skipped_ran.clone();
coordinator.register_shutdown_hook(ShutdownPhase::DrainQueue, move || {
let flag = skipped_flag.clone();
Box::pin(async move {
flag.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
})
});
assert!(
!coordinator.run_phased_shutdown().await,
"budget-exceeding shutdown must report not-ok"
);
assert_eq!(slow_ran.load(std::sync::atomic::Ordering::SeqCst), 1);
assert_eq!(
skipped_ran.load(std::sync::atomic::Ordering::SeqCst),
0,
"hook after global budget exhaustion must be skipped"
);
}
use crate::domain::models::{Task, TaskType};
use crate::domain::repositories::task_repository::{
RepositoryError, TaskQueryParams, TaskRepository,
};
use std::collections::HashSet;
struct RecordingTaskRepository {
reset_calls: std::sync::atomic::AtomicU32,
last_timeout: parking_lot::Mutex<Option<chrono::Duration>>,
}
impl RecordingTaskRepository {
fn new() -> Self {
Self {
reset_calls: std::sync::atomic::AtomicU32::new(0),
last_timeout: parking_lot::Mutex::new(None),
}
}
fn reset_calls(&self) -> u32 {
self.reset_calls.load(std::sync::atomic::Ordering::SeqCst)
}
fn last_timeout(&self) -> Option<chrono::Duration> {
*self.last_timeout.lock()
}
}
#[async_trait]
impl TaskRepository for RecordingTaskRepository {
async fn create(&self, _task: &Task) -> Result<Task, RepositoryError> {
Err(RepositoryError::NotFound)
}
async fn find_by_id(
&self,
_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn update(&self, _task: &Task) -> Result<Task, RepositoryError> {
Err(RepositoryError::NotFound)
}
async fn acquire_next(
&self,
_worker_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn mark_completed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_failed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_cancelled(&self, _id: crate::domain::id::Id) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn exists_by_url(&self, _url: &str) -> Result<bool, RepositoryError> {
Ok(false)
}
async fn find_existing_urls(
&self,
_urls: &[String],
) -> Result<HashSet<String>, RepositoryError> {
Ok(HashSet::new())
}
async fn reset_stuck_tasks(
&self,
timeout: chrono::Duration,
) -> Result<u64, RepositoryError> {
self.reset_calls
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
*self.last_timeout.lock() = Some(timeout);
Ok(0)
}
async fn cancel_tasks_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn expire_tasks(&self) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn find_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<Vec<Task>, RepositoryError> {
Ok(Vec::new())
}
async fn query_tasks(
&self,
_params: TaskQueryParams,
) -> Result<(Vec<Task>, u64), RepositoryError> {
Ok((Vec::new(), 0))
}
async fn batch_cancel(
&self,
_task_ids: Vec<crate::domain::id::Id>,
_team_id: crate::domain::id::Id,
_force: bool,
) -> Result<
(
Vec<crate::domain::id::Id>,
Vec<(crate::domain::id::Id, String)>,
),
RepositoryError,
> {
Ok((Vec::new(), Vec::new()))
}
async fn renew_lock(
&self,
_task_id: crate::domain::id::Id,
_worker_id: crate::domain::id::Id,
_extend_seconds: i64,
) -> Result<bool, RepositoryError> {
Ok(true)
}
}
#[tokio::test]
async fn test_rollback_pending_tasks_invokes_reset_with_zero_timeout() {
let concrete = Arc::new(RecordingTaskRepository::new());
let repo: Arc<dyn TaskRepository> = concrete.clone();
rollback_pending_tasks(&repo, Duration::from_millis(100), &[]).await;
assert_eq!(
concrete.reset_calls(),
1,
"rollback must invoke reset_stuck_tasks once"
);
assert_eq!(
concrete.last_timeout(),
Some(chrono::Duration::zero()),
"rollback must reset ALL in-flight tasks immediately (timeout=0)"
);
}
#[tokio::test]
async fn test_shutdown_flow_trigger_rollback_order() {
let coordinator = Arc::new(ShutdownCoordinator::new(Duration::from_millis(100)));
let repo: Arc<dyn TaskRepository> = Arc::new(RecordingTaskRepository::new());
let coord_in_task = coordinator.clone();
let worker_task = tokio::spawn(async move {
coord_in_task.wait_for_completion().await;
rollback_pending_tasks(&repo, Duration::from_millis(100), &[]).await;
});
tokio::time::sleep(Duration::from_millis(20)).await;
coordinator.trigger();
let _ = tokio::time::timeout(Duration::from_secs(5), worker_task)
.await
.expect("worker shutdown flow should complete within timeout");
assert!(
coordinator.is_shutting_down(),
"coordinator should be shutting down"
);
}
#[test]
fn test_task_type_enum_external_reference() {
let _ = TaskType::Scrape;
}
struct ErrorTaskRepository;
#[async_trait]
impl TaskRepository for ErrorTaskRepository {
async fn create(&self, _task: &Task) -> Result<Task, RepositoryError> {
Err(RepositoryError::NotFound)
}
async fn find_by_id(
&self,
_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Err(RepositoryError::NotFound)
}
async fn update(&self, _task: &Task) -> Result<Task, RepositoryError> {
Err(RepositoryError::NotFound)
}
async fn acquire_next(
&self,
_worker_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Err(RepositoryError::NotFound)
}
async fn mark_completed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_failed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_cancelled(&self, _id: crate::domain::id::Id) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn exists_by_url(&self, _url: &str) -> Result<bool, RepositoryError> {
Ok(false)
}
async fn find_existing_urls(
&self,
_urls: &[String],
) -> Result<HashSet<String>, RepositoryError> {
Ok(HashSet::new())
}
async fn reset_stuck_tasks(
&self,
_timeout: chrono::Duration,
) -> Result<u64, RepositoryError> {
Err(RepositoryError::Database(anyhow::anyhow!(
"db connection lost"
)))
}
async fn cancel_tasks_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn expire_tasks(&self) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn find_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<Vec<Task>, RepositoryError> {
Ok(vec![])
}
async fn query_tasks(
&self,
_params: TaskQueryParams,
) -> Result<(Vec<Task>, u64), RepositoryError> {
Ok((vec![], 0))
}
async fn batch_cancel(
&self,
_task_ids: Vec<crate::domain::id::Id>,
_team_id: crate::domain::id::Id,
_force: bool,
) -> Result<
(
Vec<crate::domain::id::Id>,
Vec<(crate::domain::id::Id, String)>,
),
RepositoryError,
> {
Ok((vec![], vec![]))
}
async fn renew_lock(
&self,
_task_id: crate::domain::id::Id,
_worker_id: crate::domain::id::Id,
_extend_seconds: i64,
) -> Result<bool, RepositoryError> {
Ok(true)
}
}
struct SlowTaskRepository;
#[async_trait]
impl TaskRepository for SlowTaskRepository {
async fn create(&self, _task: &Task) -> Result<Task, RepositoryError> {
Ok(Task::new(
crate::domain::id::Id::generate(),
TaskType::Scrape,
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
String::new(),
serde_json::json!({}),
))
}
async fn find_by_id(
&self,
_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn update(&self, _task: &Task) -> Result<Task, RepositoryError> {
Ok(Task::new(
crate::domain::id::Id::generate(),
TaskType::Scrape,
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
String::new(),
serde_json::json!({}),
))
}
async fn acquire_next(
&self,
_worker_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn mark_completed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_failed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_cancelled(&self, _id: crate::domain::id::Id) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn exists_by_url(&self, _url: &str) -> Result<bool, RepositoryError> {
Ok(false)
}
async fn find_existing_urls(
&self,
_urls: &[String],
) -> Result<HashSet<String>, RepositoryError> {
Ok(HashSet::new())
}
async fn reset_stuck_tasks(
&self,
_timeout: chrono::Duration,
) -> Result<u64, RepositoryError> {
tokio::time::sleep(Duration::from_secs(10)).await;
Ok(0)
}
async fn cancel_tasks_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn expire_tasks(&self) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn find_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<Vec<Task>, RepositoryError> {
Ok(vec![])
}
async fn query_tasks(
&self,
_params: TaskQueryParams,
) -> Result<(Vec<Task>, u64), RepositoryError> {
Ok((vec![], 0))
}
async fn batch_cancel(
&self,
_task_ids: Vec<crate::domain::id::Id>,
_team_id: crate::domain::id::Id,
_force: bool,
) -> Result<
(
Vec<crate::domain::id::Id>,
Vec<(crate::domain::id::Id, String)>,
),
RepositoryError,
> {
Ok((vec![], vec![]))
}
async fn renew_lock(
&self,
_task_id: crate::domain::id::Id,
_worker_id: crate::domain::id::Id,
_extend_seconds: i64,
) -> Result<bool, RepositoryError> {
Ok(true)
}
}
#[tokio::test]
async fn test_rollback_pending_tasks_handles_repository_error() {
let repo: Arc<dyn TaskRepository> = Arc::new(ErrorTaskRepository);
rollback_pending_tasks(&repo, Duration::from_secs(1), &[]).await;
}
#[tokio::test]
async fn test_rollback_pending_tasks_handles_timeout() {
let repo: Arc<dyn TaskRepository> = Arc::new(SlowTaskRepository);
rollback_pending_tasks(&repo, Duration::from_millis(10), &[]).await;
}
}