use crate::domain::models::{Task, TaskStatus};
use crate::domain::repositories::task_repository::TaskRepository;
use crate::utils::retry_policy::RetryPolicy;
use chrono::{Duration, Utc};
use log::{info, warn};
use std::sync::Arc;
pub enum HandleFailureResult {
Retried {
attempt_count: u32,
next_retry_at: chrono::DateTime<Utc>,
},
Failed,
Stale,
Error(anyhow::Error),
}
pub struct RetryHandler {
repository: Arc<dyn TaskRepository>,
retry_policy: RetryPolicy,
}
impl RetryHandler {
pub fn new(repository: Arc<dyn TaskRepository>, retry_policy: RetryPolicy) -> Self {
Self {
repository,
retry_policy,
}
}
pub fn with_default_policy(repository: Arc<dyn TaskRepository>) -> Self {
Self::new(repository, RetryPolicy::default())
}
pub async fn handle_failure(&self, task: &mut Task, error: &str) -> HandleFailureResult {
let new_attempt_count = (task.attempt_count + 1) as u32;
let effective_max_retries = self
.retry_policy
.max_retries
.min(task.max_retries.max(1) as u32);
if new_attempt_count > effective_max_retries {
return self
.terminate_task(task, error, new_attempt_count, effective_max_retries)
.await;
}
self.schedule_retry(task, new_attempt_count).await
}
async fn terminate_task(
&self,
task: &mut Task,
error: &str,
new_attempt_count: u32,
effective_max_retries: u32,
) -> HandleFailureResult {
warn!(
"Task {} exceeded max retries (effective={}, policy={}, task declared={}), marking as failed",
task.id,
effective_max_retries,
self.retry_policy.max_retries,
task.max_retries
);
task.attempt_count = new_attempt_count as i32;
task.retry_count += 1;
let snapshot_lock = task.lock_token;
let snapshot_lock_expires = task.lock_expires_at;
task.fail_with_reason(error);
task.lock_token = snapshot_lock;
task.lock_expires_at = snapshot_lock_expires;
let rows = match self.repository.update_task_guarded(task).await {
Ok(rows) => rows,
Err(e) => return HandleFailureResult::Error(e.into()),
};
if rows == 0 {
warn!(
"Task {} terminal-fail not written (guard failed: migrated by another worker)",
task.id
);
return HandleFailureResult::Stale;
}
HandleFailureResult::Failed
}
async fn schedule_retry(&self, task: &mut Task, new_attempt_count: u32) -> HandleFailureResult {
let backoff = self.retry_policy.calculate_backoff(new_attempt_count);
let next_retry = Utc::now() + Duration::milliseconds(backoff.as_millis() as i64);
task.attempt_count = new_attempt_count as i32;
task.retry_count += 1;
task.scheduled_at = Some(next_retry);
task.status = TaskStatus::Queued;
task.started_at = None;
task.completed_at = None;
let rows = match self.repository.update_task_guarded(task).await {
Ok(rows) => rows,
Err(e) => return HandleFailureResult::Error(e.into()),
};
if rows == 0 {
warn!(
"Task {} retry requeue not written (guard failed: migrated by another worker)",
task.id
);
return HandleFailureResult::Stale;
}
info!(
"Scheduled retry {}/{} for task {} in {:?}",
new_attempt_count, task.max_retries, task.id, backoff
);
HandleFailureResult::Retried {
attempt_count: new_attempt_count,
next_retry_at: next_retry,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::domain::models::{Task, TaskStatus, TaskType};
use crate::domain::repositories::task_repository::{
RepositoryError, TaskQueryParams, TaskRepository,
};
use crate::utils::retry_policy::RetryPolicy;
use async_trait::async_trait;
use std::collections::HashSet;
use std::sync::Arc;
use std::sync::Mutex;
struct MockTaskRepository {
last_updated: Mutex<Option<Task>>,
fail_update: Mutex<bool>,
}
impl MockTaskRepository {
fn new() -> Self {
Self {
last_updated: Mutex::new(None),
fail_update: Mutex::new(false),
}
}
fn set_fail_update(&self, fail: bool) {
*self.fail_update.lock().expect("lock fail_update") = fail;
}
fn last_updated_task(&self) -> Option<Task> {
self.last_updated.lock().expect("lock last_updated").clone()
}
}
#[async_trait]
impl TaskRepository for MockTaskRepository {
async fn create(&self, _task: &Task) -> Result<Task, RepositoryError> {
Err(RepositoryError::Database(anyhow::anyhow!(
"not implemented"
)))
}
async fn find_by_id(
&self,
_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn update(&self, task: &Task) -> Result<Task, RepositoryError> {
if *self.fail_update.lock().expect("lock fail_update") {
return Err(RepositoryError::Database(anyhow::anyhow!(
"mock update failure"
)));
}
*self.last_updated.lock().expect("lock last_updated") = Some(task.clone());
Ok(task.clone())
}
async fn update_task_guarded(&self, task: &Task) -> Result<u64, RepositoryError> {
if *self.fail_update.lock().expect("lock fail_update") {
return Err(RepositoryError::Database(anyhow::anyhow!(
"mock update failure"
)));
}
*self.last_updated.lock().expect("lock last_updated") = Some(task.clone());
Ok(1)
}
async fn acquire_next(
&self,
_worker_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn mark_completed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_failed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_cancelled(&self, _id: crate::domain::id::Id) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn exists_by_url(&self, _url: &str) -> Result<bool, RepositoryError> {
Ok(false)
}
async fn find_existing_urls(
&self,
_urls: &[String],
) -> Result<HashSet<String>, RepositoryError> {
Ok(HashSet::new())
}
async fn reset_stuck_tasks(
&self,
_timeout: chrono::Duration,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn cancel_tasks_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn expire_tasks(&self) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn find_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<Vec<Task>, RepositoryError> {
Ok(vec![])
}
async fn query_tasks(
&self,
_params: TaskQueryParams,
) -> Result<(Vec<Task>, u64), RepositoryError> {
Ok((vec![], 0))
}
async fn batch_cancel(
&self,
_task_ids: Vec<crate::domain::id::Id>,
_team_id: crate::domain::id::Id,
_force: bool,
) -> Result<
(
Vec<crate::domain::id::Id>,
Vec<(crate::domain::id::Id, String)>,
),
RepositoryError,
> {
Ok((vec![], vec![]))
}
async fn renew_lock(
&self,
_task_id: crate::domain::id::Id,
_worker_id: crate::domain::id::Id,
_extend_seconds: i64,
) -> Result<bool, RepositoryError> {
Ok(true)
}
}
fn make_task(attempt_count: i32, max_retries: i32) -> Task {
let mut task = Task::new(
crate::domain::id::Id::generate(),
TaskType::Scrape,
crate::domain::id::Id::generate(),
crate::domain::id::Id::generate(),
"https://example.com".to_string(),
serde_json::json!({}),
);
task.attempt_count = attempt_count;
task.max_retries = max_retries;
task
}
#[test]
fn test_new_stores_repository_and_policy() {
let repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository::new());
let policy = RetryPolicy::fast();
let handler = RetryHandler::new(repo.clone(), policy.clone());
let _ = &handler;
}
#[test]
fn test_with_default_policy_uses_default_retry_policy() {
let repo: Arc<dyn TaskRepository> = Arc::new(MockTaskRepository::new());
let handler = RetryHandler::with_default_policy(repo.clone());
let _ = &handler;
}
#[tokio::test]
async fn test_handle_failure_retries_when_below_max() {
let mock = Arc::new(MockTaskRepository::new());
let repo: Arc<dyn TaskRepository> = mock.clone();
let policy = RetryPolicy {
enable_jitter: false,
..Default::default()
};
let handler = RetryHandler::new(repo, policy);
let before = Utc::now();
let mut task = make_task(0, 5);
let result = handler.handle_failure(&mut task, "boom").await;
match result {
HandleFailureResult::Retried {
attempt_count,
next_retry_at,
} => {
assert_eq!(attempt_count, 1, "attempt_count should be incremented to 1");
assert!(
next_retry_at > before,
"next_retry_at should be in the future"
);
}
_ => panic!("expected Retried, got different variant"),
}
assert_eq!(task.attempt_count, 1);
assert_eq!(task.retry_count, 1);
assert_eq!(task.status, TaskStatus::Queued);
assert!(task.scheduled_at.is_some(), "scheduled_at should be set");
assert!(task.started_at.is_none(), "started_at should be cleared");
assert!(
task.completed_at.is_none(),
"completed_at should be cleared"
);
let updated = mock
.last_updated_task()
.expect("update should have been called");
assert_eq!(updated.id, task.id);
assert_eq!(updated.status, TaskStatus::Queued);
}
#[tokio::test]
async fn test_handle_failure_retries_increments_attempt_count_correctly() {
let mock = Arc::new(MockTaskRepository::new());
let repo: Arc<dyn TaskRepository> = mock.clone();
let policy = RetryPolicy {
enable_jitter: false,
..Default::default()
};
let handler = RetryHandler::new(repo, policy);
let mut task = make_task(2, 5);
let result = handler.handle_failure(&mut task, "boom").await;
match result {
HandleFailureResult::Retried { attempt_count, .. } => {
assert_eq!(
attempt_count, 3,
"attempt_count should be 3 after increment"
);
}
_ => panic!("expected Retried, got different variant"),
}
assert_eq!(task.attempt_count, 3);
assert_eq!(task.retry_count, 1);
}
#[tokio::test]
async fn test_handle_failure_fails_when_max_retries_exceeded() {
let mock = Arc::new(MockTaskRepository::new());
let repo: Arc<dyn TaskRepository> = mock.clone();
let policy = RetryPolicy {
max_retries: 0,
..Default::default()
};
let handler = RetryHandler::new(repo, policy);
let before = Utc::now();
let mut task = make_task(0, 5);
let result = handler.handle_failure(&mut task, "boom").await;
match result {
HandleFailureResult::Failed => { }
_ => panic!("expected Failed, got different variant"),
}
assert_eq!(task.attempt_count, 1);
assert_eq!(task.retry_count, 1);
assert_eq!(task.status, TaskStatus::Failed);
assert!(
task.completed_at.is_some(),
"completed_at should be set on failure"
);
assert!(
task.completed_at.expect("completed_at set") >= before,
"completed_at should be ~now"
);
let updated = mock
.last_updated_task()
.expect("update should have been called");
assert_eq!(updated.status, TaskStatus::Failed);
}
#[tokio::test]
async fn test_handle_failure_fails_when_attempt_reaches_max() {
let mock = Arc::new(MockTaskRepository::new());
let repo: Arc<dyn TaskRepository> = mock.clone();
let handler = RetryHandler::new(repo, RetryPolicy::default());
let mut task = make_task(3, 3);
let result = handler.handle_failure(&mut task, "boom").await;
assert!(
matches!(result, HandleFailureResult::Failed),
"should fail when attempt_count exceeds task.max_retries"
);
assert_eq!(task.status, TaskStatus::Failed);
assert_eq!(task.attempt_count, 4);
}
#[tokio::test]
async fn test_handle_failure_terminal_writes_payload_error() {
let mock = Arc::new(MockTaskRepository::new());
let repo: Arc<dyn TaskRepository> = mock.clone();
let policy = RetryPolicy {
max_retries: 0,
..Default::default()
};
let handler = RetryHandler::new(repo, policy);
let mut task = make_task(0, 5);
let result = handler
.handle_failure(&mut task, "connection timeout")
.await;
assert!(matches!(result, HandleFailureResult::Failed));
assert_eq!(
task.payload.get("error").and_then(|v| v.as_str()),
Some("connection timeout"),
"terminal path must record error string in payload"
);
let updated = mock
.last_updated_task()
.expect("update should have been called");
assert_eq!(
updated.payload.get("error").and_then(|v| v.as_str()),
Some("connection timeout")
);
}
#[tokio::test]
async fn test_handle_failure_retry_path_does_not_write_payload_error() {
let mock = Arc::new(MockTaskRepository::new());
let repo: Arc<dyn TaskRepository> = mock.clone();
let policy = RetryPolicy {
enable_jitter: false,
..Default::default()
};
let handler = RetryHandler::new(repo, policy);
let mut task = make_task(0, 5);
let result = handler.handle_failure(&mut task, "transient").await;
assert!(matches!(result, HandleFailureResult::Retried { .. }));
assert!(
task.payload.get("error").is_none(),
"retry path must not write payload.error"
);
}
#[tokio::test]
async fn test_handle_failure_terminal_empty_error_writes_nothing() {
let mock = Arc::new(MockTaskRepository::new());
let repo: Arc<dyn TaskRepository> = mock.clone();
let policy = RetryPolicy {
max_retries: 0,
..Default::default()
};
let handler = RetryHandler::new(repo, policy);
let mut task = make_task(0, 5);
let result = handler.handle_failure(&mut task, "").await;
assert!(matches!(result, HandleFailureResult::Failed));
assert!(
task.payload.get("error").is_none(),
"empty error must not create payload.error"
);
}
#[tokio::test]
async fn test_handle_failure_policy_cap_cannot_exceed_task_max_retries() {
let mock = Arc::new(MockTaskRepository::new());
let repo: Arc<dyn TaskRepository> = mock.clone();
let handler = RetryHandler::new(repo, RetryPolicy::default());
let mut task = make_task(2, 2);
let result = handler.handle_failure(&mut task, "boom").await;
assert!(
matches!(result, HandleFailureResult::Failed),
"task.max_retries=2 must cap retries below policy max_retries=5"
);
assert_eq!(task.status, TaskStatus::Failed);
}
#[tokio::test]
async fn test_handle_failure_returns_error_when_update_fails_on_retry() {
let mock = Arc::new(MockTaskRepository::new());
mock.set_fail_update(true);
let repo: Arc<dyn TaskRepository> = mock.clone();
let handler = RetryHandler::new(repo, RetryPolicy::default());
let mut task = make_task(0, 5);
let result = handler.handle_failure(&mut task, "boom").await;
match result {
HandleFailureResult::Error(err) => {
assert!(
err.to_string().contains("mock update failure"),
"error should contain mock message: {}",
err
);
}
_ => panic!("expected Error, got different variant"),
}
assert_eq!(task.attempt_count, 1);
assert_eq!(task.retry_count, 1);
assert_eq!(task.status, TaskStatus::Queued);
}
#[tokio::test]
async fn test_handle_failure_returns_error_when_update_fails_on_fail() {
let mock = Arc::new(MockTaskRepository::new());
mock.set_fail_update(true);
let repo: Arc<dyn TaskRepository> = mock.clone();
let policy = RetryPolicy {
max_retries: 0,
..Default::default()
};
let handler = RetryHandler::new(repo, policy);
let mut task = make_task(0, 5);
let result = handler.handle_failure(&mut task, "boom").await;
match result {
HandleFailureResult::Error(err) => {
assert!(
err.to_string().contains("mock update failure"),
"error should contain mock message: {}",
err
);
}
_ => panic!("expected Error, got different variant"),
}
assert_eq!(task.status, TaskStatus::Failed);
assert_eq!(task.attempt_count, 1);
assert!(task.completed_at.is_some());
}
#[test]
fn test_handle_failure_result_reried_carries_correct_fields() {
let now = Utc::now();
let result = HandleFailureResult::Retried {
attempt_count: 3,
next_retry_at: now,
};
match result {
HandleFailureResult::Retried {
attempt_count,
next_retry_at,
} => {
assert_eq!(attempt_count, 3);
assert_eq!(next_retry_at, now);
}
_ => panic!("wrong variant"),
}
}
#[test]
fn test_handle_failure_result_failed_is_unit() {
let result = HandleFailureResult::Failed;
assert!(matches!(result, HandleFailureResult::Failed));
}
#[test]
fn test_handle_failure_result_error_carries_message() {
let result = HandleFailureResult::Error(anyhow::anyhow!("boom"));
match result {
HandleFailureResult::Error(e) => assert!(e.to_string().contains("boom")),
_ => panic!("wrong variant"),
}
}
#[tokio::test]
async fn test_mock_task_repository_stub_methods_return_expected_defaults() {
let mock = MockTaskRepository::new();
let task = make_task(0, 5);
assert!(mock.create(&task).await.is_err());
assert!(mock.find_by_id(task.id).await.unwrap().is_none());
assert!(mock
.acquire_next(crate::domain::id::Id::generate())
.await
.unwrap()
.is_none());
assert!(!mock.exists_by_url("https://example.com").await.unwrap());
assert!(mock
.find_existing_urls(&["https://example.com".to_string()])
.await
.unwrap()
.is_empty());
assert!(mock
.find_by_crawl_id(crate::domain::id::Id::generate())
.await
.unwrap()
.is_empty());
mock.mark_completed(task.id, None).await.unwrap();
mock.mark_failed(task.id, None).await.unwrap();
mock.mark_cancelled(task.id).await.unwrap();
assert_eq!(
mock.reset_stuck_tasks(Duration::seconds(60)).await.unwrap(),
0
);
assert_eq!(
mock.cancel_tasks_by_crawl_id(crate::domain::id::Id::generate())
.await
.unwrap(),
0
);
assert_eq!(mock.expire_tasks().await.unwrap(), 0);
let (success, failed) = mock
.batch_cancel(vec![task.id], task.team_id, false)
.await
.unwrap();
assert!(success.is_empty());
assert!(failed.is_empty());
}
}